From a95456d055769bf688a22d73df5a8b9f30400339 Mon Sep 17 00:00:00 2001 From: Victor Gaydov Date: Mon, 16 Sep 2024 13:49:17 +0700 Subject: [PATCH] Android backend overhaul and fixes (#98) Also fixes: #1, #16, #17, #66 - Initiate request for notification and microphone permissions and media projection on the dart side of AndroidBackend. - Implement those requests in MainActivity.kt and rework AndroidConnector to redirect those requests to activity. - Automatically stop media projection when both sender and receiver are stopped. Prevent service from stopping projection while we are starting sender and receiver. - Implement synchronization and fix various races. - Use foreground service instead of bound service, to keep it running when app closes. - Forbid swiping away notification on lock screen. (We can forbid it only for lock screen). - Stop sender and receiver when notification is swiped away. - Add AndroidListener, to pass events from kotlin to dart. Implement in in AndroidBackend on dart side. - Add AndroidSenderSettings, AndroidReceiverSettings, and pass them from model to kotlin. - Hard-code ports in model instead of kotlin. - Implement Backend.getLocalAddresses(). - Improve comments. - Refactor android service code. - Remove unused values from strings.xml. --- android/app/build.gradle | 14 +- android/app/src/main/AndroidManifest.xml | 17 +- .../connector/AndroidConnector.g.kt | 380 ++++++++- .../connector/AndroidConnectorImpl.kt | 227 +++++- .../org/rocstreaming/rocdroid/MainActivity.kt | 238 +++++- .../service/SenderReceiverService.kt | 766 +++++++++++------- android/app/src/main/res/values/strings.xml | 47 +- dodo.py | 18 +- l10n.yaml | 2 +- lib/src/agent.dart | 2 + lib/src/agent/android_backend.dart | 143 +++- lib/src/agent/android_connector.dart | 147 +++- lib/src/agent/android_connector.g.dart | 398 ++++++++- lib/src/agent/backend.dart | 30 +- lib/src/model/model_root.dart | 5 +- lib/src/model/receiver.dart | 7 +- lib/src/model/sender.dart | 13 +- 17 files changed, 2037 insertions(+), 417 deletions(-) diff --git a/android/app/build.gradle b/android/app/build.gradle index bbe3f74..6d6784d 100644 --- a/android/app/build.gradle +++ b/android/app/build.gradle @@ -44,7 +44,8 @@ android { defaultConfig { applicationId = "org.rocstreaming.rocdroid" // You can update the following values to match your application needs. - // For more information, see: https://docs.flutter.dev/deployment/android#reviewing-the-gradle-build-configuration. + // For more information, see: + // https://docs.flutter.dev/deployment/android#reviewing-the-gradle-build-configuration minSdk = flutter.minSdkVersion targetSdk = flutter.targetSdkVersion versionCode = flutterVersionCode.toInteger() @@ -65,6 +66,15 @@ android { signingConfig = signingConfigs.debug } } + + // https://android.izzysoft.de/articles/named/iod-scan-apkchecks#blobs + // https://gist.github.com/obfusk/31c332b884464cd8aa06ce1ba1583c05 + dependenciesInfo { + // Disables dependency metadata when building APKs. + includeInApk = false + // Disables dependency metadata when building Android App Bundles. + includeInBundle = false + } } flutter { @@ -76,7 +86,7 @@ dependencies { implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk8:$kotlin_version" implementation 'androidx.appcompat:appcompat:1.1.0' implementation 'androidx.core:core-ktx:1.3.0' - implementation 'androidx.activity:activity-ktx:1.2.4' + implementation 'androidx.activity:activity-ktx:1.9.2' implementation 'androidx.fragment:fragment-ktx:1.3.6' implementation 'androidx.constraintlayout:constraintlayout:1.1.3' implementation 'com.google.android.material:material:1.3.0' diff --git a/android/app/src/main/AndroidManifest.xml b/android/app/src/main/AndroidManifest.xml index c792549..d7f1759 100644 --- a/android/app/src/main/AndroidManifest.xml +++ b/android/app/src/main/AndroidManifest.xml @@ -1,17 +1,29 @@ + android:versionCode="3000" + android:versionName="0.3.0"> + + + + + + + + + diff --git a/android/app/src/main/kotlin/org/rocstreaming/connector/AndroidConnector.g.kt b/android/app/src/main/kotlin/org/rocstreaming/connector/AndroidConnector.g.kt index 1415270..b33afd4 100644 --- a/android/app/src/main/kotlin/org/rocstreaming/connector/AndroidConnector.g.kt +++ b/android/app/src/main/kotlin/org/rocstreaming/connector/AndroidConnector.g.kt @@ -31,6 +31,9 @@ private fun wrapError(exception: Throwable): List { } } +private fun createConnectionError(channelName: String): FlutterError { + return FlutterError("channel-error", "Unable to establish connection on channel: '$channelName'.", "")} + /** * Error class for passing custom error details to Flutter via a thrown PlatformException. * @property code The error code. @@ -42,43 +45,336 @@ class FlutterError ( override val message: String? = null, val details: Any? = null ) : Throwable() -private open class AndroidConnectorPigeonCodec : StandardMessageCodec() { + +/** Where sender gets sound. */ +enum class AndroidCaptureType(val raw: Int) { + /** Capture from locally playing apps. */ + CAPTURE_APPS(0), + /** Capture from local microphone. */ + CAPTURE_MIC(1); + + companion object { + fun ofRaw(raw: Int): AndroidCaptureType? { + return values().firstOrNull { it.raw == raw } + } + } +} + +/** Asynchronous events produces by android service. */ +enum class AndroidServiceEvent(val raw: Int) { + SENDER_STATE_CHANGED(0), + RECEIVER_STATE_CHANGED(1); + + companion object { + fun ofRaw(raw: Int): AndroidServiceEvent? { + return values().firstOrNull { it.raw == raw } + } + } +} + +/** Asynchronous errors produces by android service. */ +enum class AndroidServiceError(val raw: Int) { + AUDIO_RECORD_FAILED(0), + AUDIO_TRACK_FAILED(1), + SENDER_CONNECT_FAILED(2), + RECEIVER_BIND_FAILED(3); + + companion object { + fun ofRaw(raw: Int): AndroidServiceError? { + return values().firstOrNull { it.raw == raw } + } + } +} + +/** + * Receiver settings. + * + * Generated class from Pigeon that represents data sent in messages. + */ +data class AndroidReceiverSettings ( + /** Local port to receive source packets. */ + val sourcePort: Long, + /** Local port to receive repair packets. */ + val repairPort: Long +) + { + companion object { + fun fromList(pigeonVar_list: List): AndroidReceiverSettings { + val sourcePort = pigeonVar_list[0] as Long + val repairPort = pigeonVar_list[1] as Long + return AndroidReceiverSettings(sourcePort, repairPort) + } + } + fun toList(): List { + return listOf( + sourcePort, + repairPort, + ) + } +} + +/** + * Sender settings. + * + * Generated class from Pigeon that represents data sent in messages. + */ +data class AndroidSenderSettings ( + /** From where to capture stream. */ + val captureType: AndroidCaptureType, + /** IP address or hostname where to send packets. */ + val host: String, + /** Remote port where to send source packets. */ + val sourcePort: Long, + /** Remote port where to send repair packets. */ + val repairPort: Long +) + { + companion object { + fun fromList(pigeonVar_list: List): AndroidSenderSettings { + val captureType = pigeonVar_list[0] as AndroidCaptureType + val host = pigeonVar_list[1] as String + val sourcePort = pigeonVar_list[2] as Long + val repairPort = pigeonVar_list[3] as Long + return AndroidSenderSettings(captureType, host, sourcePort, repairPort) + } + } + fun toList(): List { + return listOf( + captureType, + host, + sourcePort, + repairPort, + ) + } +} +private open class AndroidBridgePigeonCodec : StandardMessageCodec() { override fun readValueOfType(type: Byte, buffer: ByteBuffer): Any? { - return super.readValueOfType(type, buffer) + return when (type) { + 129.toByte() -> { + return (readValue(buffer) as Long?)?.let { + AndroidCaptureType.ofRaw(it.toInt()) + } + } + 130.toByte() -> { + return (readValue(buffer) as Long?)?.let { + AndroidServiceEvent.ofRaw(it.toInt()) + } + } + 131.toByte() -> { + return (readValue(buffer) as Long?)?.let { + AndroidServiceError.ofRaw(it.toInt()) + } + } + 132.toByte() -> { + return (readValue(buffer) as? List)?.let { + AndroidReceiverSettings.fromList(it) + } + } + 133.toByte() -> { + return (readValue(buffer) as? List)?.let { + AndroidSenderSettings.fromList(it) + } + } + else -> super.readValueOfType(type, buffer) + } } override fun writeValue(stream: ByteArrayOutputStream, value: Any?) { - super.writeValue(stream, value) + when (value) { + is AndroidCaptureType -> { + stream.write(129) + writeValue(stream, value.raw) + } + is AndroidServiceEvent -> { + stream.write(130) + writeValue(stream, value.raw) + } + is AndroidServiceError -> { + stream.write(131) + writeValue(stream, value.raw) + } + is AndroidReceiverSettings -> { + stream.write(132) + writeValue(stream, value.toList()) + } + is AndroidSenderSettings -> { + stream.write(133) + writeValue(stream, value.toList()) + } + else -> super.writeValue(stream, value) + } } } + /** - * ???. + * Allows to invoke kotlin methods from dart. + * + * This declaration emits 2 classes: + * dart: AndroidConnector implementation class, which methods invoke kotlin + * methods under the hood (via platform channels) + * kotlin: AndroidConnector interface, which we implement in + * AndroidConnectorImpl, where the actual work is done * * Generated interface from Pigeon that represents a handler of messages from Flutter. */ interface AndroidConnector { - fun startReceiver() + /** Get list of IP addresses of available network interfaces. */ + fun getLocalAddresses(): List + /** + * Request permission to post notifications, if no already granted. + * Must be called before acquiring projection first time. + * If returns false, user rejected permission and notifications won't appear. + */ + fun requestNotifications(callback: (Result) -> Unit) + /** + * Request permission to capture local microphone, if not already granted. + * Must be called before starting sender when using AndroidCaptureType.captureMic. + * If returns false, user rejected permission and sender won't start. + */ + fun requestMicrophone(callback: (Result) -> Unit) + /** + * Request access to media projection, if not already granted. + * Must be called before starting sender or receiver. + * If returns false, user rejected access and sender/receiver won't start. + * Throws exception if: + * - lost connection to foreground service + */ + fun acquireProjection(callback: (Result) -> Unit) + /** + * Allow service to stop projection when it's not needed. + * Must be called after *starting* sender or receiver. + */ + fun releaseProjection() + /** + * Start receiver. + * Receiver gets stream from network and plays to local speakers. + * Must be called between acquireProjection() and releaseProjection(). + * Throws exception if: + * - lost connection to foreground service + * - media projection wasn't acquired + */ + fun startReceiver(settings: AndroidReceiverSettings) + /** Stop receiver. */ fun stopReceiver() + /** Check if receiver is running. */ fun isReceiverAlive(): Boolean - fun startSender(ip: String) + /** + * Start sender. + * Sender gets stream from local microphone OR media system apps, and streams to network. + * Must be called between acquireProjection() and releaseProjection(). + * Throws exception if: + * - lost connection to foreground service + * - media projection not acquired + * - microphone permission is needed and wasn't granted + */ + fun startSender(settings: AndroidSenderSettings) + /** Stop sender. */ fun stopSender() + /** Check if sender is running. */ fun isSenderAlive(): Boolean companion object { /** The codec used by AndroidConnector. */ val codec: MessageCodec by lazy { - AndroidConnectorPigeonCodec() + AndroidBridgePigeonCodec() } /** Sets up an instance of `AndroidConnector` to handle messages through the `binaryMessenger`. */ @JvmOverloads fun setUp(binaryMessenger: BinaryMessenger, api: AndroidConnector?, messageChannelSuffix: String = "") { val separatedMessageChannelSuffix = if (messageChannelSuffix.isNotEmpty()) ".$messageChannelSuffix" else "" run { - val channel = BasicMessageChannel(binaryMessenger, "dev.flutter.pigeon.roc_droid.AndroidConnector.startReceiver$separatedMessageChannelSuffix", codec) + val channel = BasicMessageChannel(binaryMessenger, "dev.flutter.pigeon.roc_droid.AndroidConnector.getLocalAddresses$separatedMessageChannelSuffix", codec) + if (api != null) { + channel.setMessageHandler { _, reply -> + val wrapped: List = try { + listOf(api.getLocalAddresses()) + } catch (exception: Throwable) { + wrapError(exception) + } + reply.reply(wrapped) + } + } else { + channel.setMessageHandler(null) + } + } + run { + val channel = BasicMessageChannel(binaryMessenger, "dev.flutter.pigeon.roc_droid.AndroidConnector.requestNotifications$separatedMessageChannelSuffix", codec) + if (api != null) { + channel.setMessageHandler { _, reply -> + api.requestNotifications{ result: Result -> + val error = result.exceptionOrNull() + if (error != null) { + reply.reply(wrapError(error)) + } else { + val data = result.getOrNull() + reply.reply(wrapResult(data)) + } + } + } + } else { + channel.setMessageHandler(null) + } + } + run { + val channel = BasicMessageChannel(binaryMessenger, "dev.flutter.pigeon.roc_droid.AndroidConnector.requestMicrophone$separatedMessageChannelSuffix", codec) + if (api != null) { + channel.setMessageHandler { _, reply -> + api.requestMicrophone{ result: Result -> + val error = result.exceptionOrNull() + if (error != null) { + reply.reply(wrapError(error)) + } else { + val data = result.getOrNull() + reply.reply(wrapResult(data)) + } + } + } + } else { + channel.setMessageHandler(null) + } + } + run { + val channel = BasicMessageChannel(binaryMessenger, "dev.flutter.pigeon.roc_droid.AndroidConnector.acquireProjection$separatedMessageChannelSuffix", codec) + if (api != null) { + channel.setMessageHandler { _, reply -> + api.acquireProjection{ result: Result -> + val error = result.exceptionOrNull() + if (error != null) { + reply.reply(wrapError(error)) + } else { + val data = result.getOrNull() + reply.reply(wrapResult(data)) + } + } + } + } else { + channel.setMessageHandler(null) + } + } + run { + val channel = BasicMessageChannel(binaryMessenger, "dev.flutter.pigeon.roc_droid.AndroidConnector.releaseProjection$separatedMessageChannelSuffix", codec) if (api != null) { channel.setMessageHandler { _, reply -> val wrapped: List = try { - api.startReceiver() + api.releaseProjection() + listOf(null) + } catch (exception: Throwable) { + wrapError(exception) + } + reply.reply(wrapped) + } + } else { + channel.setMessageHandler(null) + } + } + run { + val channel = BasicMessageChannel(binaryMessenger, "dev.flutter.pigeon.roc_droid.AndroidConnector.startReceiver$separatedMessageChannelSuffix", codec) + if (api != null) { + channel.setMessageHandler { message, reply -> + val args = message as List + val settingsArg = args[0] as AndroidReceiverSettings + val wrapped: List = try { + api.startReceiver(settingsArg) listOf(null) } catch (exception: Throwable) { wrapError(exception) @@ -125,9 +421,9 @@ interface AndroidConnector { if (api != null) { channel.setMessageHandler { message, reply -> val args = message as List - val ipArg = args[0] as String + val settingsArg = args[0] as AndroidSenderSettings val wrapped: List = try { - api.startSender(ipArg) + api.startSender(settingsArg) listOf(null) } catch (exception: Throwable) { wrapError(exception) @@ -172,3 +468,65 @@ interface AndroidConnector { } } } +/** + * Allows to invoke dart methods from kotlin. + * + * This declaration emits 2 classes: + * dart: AndroidListener interface class, which is implemented + * by AndroidBackend + * kotlin: AndroidListener implementation class, which methods invoke + * dart methods under the hood (via platform channels) + * + * Generated class from Pigeon that represents Flutter messages that can be called from Kotlin. + */ +class AndroidListener(private val binaryMessenger: BinaryMessenger, private val messageChannelSuffix: String = "") { + companion object { + /** The codec used by AndroidListener. */ + val codec: MessageCodec by lazy { + AndroidBridgePigeonCodec() + } + } + /** + * Invoked when an asynchronous event occurs. + * For example, sender is started or stopped by UI, notification button, + * tile button, or because of failure. + */ + fun onEvent(eventCodeArg: AndroidServiceEvent, callback: (Result) -> Unit) +{ + val separatedMessageChannelSuffix = if (messageChannelSuffix.isNotEmpty()) ".$messageChannelSuffix" else "" + val channelName = "dev.flutter.pigeon.roc_droid.AndroidListener.onEvent$separatedMessageChannelSuffix" + val channel = BasicMessageChannel(binaryMessenger, channelName, codec) + channel.send(listOf(eventCodeArg)) { + if (it is List<*>) { + if (it.size > 1) { + callback(Result.failure(FlutterError(it[0] as String, it[1] as String, it[2] as String?))) + } else { + callback(Result.success(Unit)) + } + } else { + callback(Result.failure(createConnectionError(channelName))) + } + } + } + /** + * Invoked when an asynchronous error occurs. + * For example, sender encounters network error. + */ + fun onError(errorCodeArg: AndroidServiceError, callback: (Result) -> Unit) +{ + val separatedMessageChannelSuffix = if (messageChannelSuffix.isNotEmpty()) ".$messageChannelSuffix" else "" + val channelName = "dev.flutter.pigeon.roc_droid.AndroidListener.onError$separatedMessageChannelSuffix" + val channel = BasicMessageChannel(binaryMessenger, channelName, codec) + channel.send(listOf(errorCodeArg)) { + if (it is List<*>) { + if (it.size > 1) { + callback(Result.failure(FlutterError(it[0] as String, it[1] as String, it[2] as String?))) + } else { + callback(Result.success(Unit)) + } + } else { + callback(Result.failure(createConnectionError(channelName))) + } + } + } +} diff --git a/android/app/src/main/kotlin/org/rocstreaming/connector/AndroidConnectorImpl.kt b/android/app/src/main/kotlin/org/rocstreaming/connector/AndroidConnectorImpl.kt index e4d9208..3db737d 100644 --- a/android/app/src/main/kotlin/org/rocstreaming/connector/AndroidConnectorImpl.kt +++ b/android/app/src/main/kotlin/org/rocstreaming/connector/AndroidConnectorImpl.kt @@ -1,41 +1,234 @@ -package org.rocstreaming.connector +package org.rocstreaming.rocdroid import AndroidConnector +import AndroidReceiverSettings +import AndroidSenderSettings +import FlutterError +import android.Manifest +import android.media.projection.MediaProjection import android.util.Log -import org.rocstreaming.service.SenderReceiverService +import java.net.NetworkInterface +import org.rocstreaming.rocdroid.MainActivity +import org.rocstreaming.rocdroid.StreamingService -private const val LOG_TAG = "[rocdroid.Connector]" +private const val BAD_SEQUENCE_CODE = "rocdroid.BAD_SEQUENCE" +private const val BAD_SEQUENCE_TEXT = "Invalid method invocation sequence" +private const val NO_SERVICE_CODE = "rocdroid.NO_SERVICE" +private const val NO_SERVICE_TEXT = "Lost connection to streaming service" + +private const val NO_PROJECTION_CODE = "rocdroid.NO_PROJECTION" +private const val NO_PROJECTION_TEXT = "Media projection wasn't acquired" + +private const val NO_PERMISSION_CODE = "rocdroid.NO_PERMISSION" +private const val NO_PERMISSION_TEXT = "Microhpone permission wasn't granted" + +private const val LOG_TAG = "rocdroid.AndroidConnectorImpl" + +// Implementation of generated interface AndroidConnector, which methods are invoked +// from the dart side. class AndroidConnectorImpl : AndroidConnector { - companion object { - lateinit var senderReceiverService : SenderReceiverService + private var projectionAcquired: Boolean = false + + fun getActivity() : MainActivity { + return MainActivity.instance + } + + override fun getLocalAddresses(): List { + try { + return NetworkInterface.getNetworkInterfaces().toList() + .flatMap { it.inetAddresses.toList() } + .filter { !it.isLoopbackAddress && !it.hostAddress.contains(':') } + .map { it.hostAddress } + } catch (e: Exception) { + Log.e(LOG_TAG, "Failed to retrieve address list: " + e.toString()) + return emptyList() + } + } + + override fun requestNotifications(callback: (Result) -> Unit) { + Log.i(LOG_TAG, "Requesting POST_NOTIFICATIONS permission") + + getActivity().requestPermission( + Manifest.permission.POST_NOTIFICATIONS, + R.string.allow_notifications_title, + R.string.allow_notifications_message, + { isGranted: Boolean -> + if (!isGranted) { + Log.w(LOG_TAG, "Permission request failed") + callback(Result.success(false)) + return@requestPermission + } + + Log.d(LOG_TAG, "Permission request succeeded") + callback(Result.success(true)) + }) + } + + override fun requestMicrophone(callback: (Result) -> Unit) { + Log.i(LOG_TAG, "Requesting RECORD_AUDIO permission") + + getActivity().requestPermission( + Manifest.permission.RECORD_AUDIO, + R.string.allow_mic_title, + R.string.allow_mic_message, + { isGranted: Boolean -> + if (!isGranted) { + Log.w(LOG_TAG, "Permission request failed") + callback(Result.success(false)) + return@requestPermission + } + + Log.d(LOG_TAG, "Permission request succeeded") + callback(Result.success(true)) + }) + } + + override fun acquireProjection(callback: (Result) -> Unit) { + Log.i(LOG_TAG, "Acquiring media projection") + + if (projectionAcquired) { + Log.e(LOG_TAG, "Unpaired acquireProjection/releaseProjection calls") + callback(Result.failure(FlutterError(BAD_SEQUENCE_CODE, BAD_SEQUENCE_TEXT))) + return + } + + projectionAcquired = true + + // If service isn't started yet, start it and invoke callback when we've connected. + // If service is already started, invoke callback immediately. + getActivity().startService({ service: StreamingService -> + // Ensure that service won't detach projection until releaseProjection(). + service.disableAutoDetach() + + if (service.hasProjection()) { + Log.d(LOG_TAG, "Projection already acquired") + callback(Result.success(true)) + return@startService + } + + getActivity().requestProjection({ projection: MediaProjection? -> + if (projection == null) { + Log.w(LOG_TAG, "Projection request failed") + callback(Result.success(false)) + return@requestProjection + } + + Log.d(LOG_TAG, "Projection request succeeded") + service.attachProjection(projection) + callback(Result.success(true)) + }) + }) } - override fun startReceiver() { - Log.d(LOG_TAG, "Try start Receiver") - senderReceiverService.startReceiver() + override fun releaseProjection() { + Log.i(LOG_TAG, "Releasing media projection") + + if (!projectionAcquired) { + Log.e(LOG_TAG, "Unpaired acquireProjection/releaseProjection calls") + throw FlutterError(BAD_SEQUENCE_CODE, BAD_SEQUENCE_TEXT) + } + + projectionAcquired = false + + val service = getActivity().getService() + if (service == null) { + return + } + + // Allow service to detach projection when it's not needed. + service.enableAutoDetach() + } + + override fun startReceiver(settings: AndroidReceiverSettings) { + Log.i(LOG_TAG, "Starting receiver if needed") + + if (!projectionAcquired) { + Log.e(LOG_TAG, "startReceiver must be called between acquireProjection/releaseProjection") + throw FlutterError(BAD_SEQUENCE_CODE, BAD_SEQUENCE_TEXT) + } + + val service = getActivity().getService() + if (service == null) { + Log.e(LOG_TAG, "Lost connection to service") + throw FlutterError(NO_SERVICE_CODE, NO_SERVICE_TEXT) + } + + if (!service.hasProjection()) { + Log.e(LOG_TAG, "Lost acquired projection") + throw FlutterError(NO_PROJECTION_CODE, NO_PROJECTION_TEXT) + } + + service.startReceiver(settings) } override fun stopReceiver() { - Log.d(LOG_TAG, "Try stop Receiver") - senderReceiverService.stopReceiver() + Log.i(LOG_TAG, "Stopping receiver if needed") + + val service = getActivity().getService() + if (service == null) { + Log.d(LOG_TAG, "Lost connection to service") + return + } + + service.stopReceiver() } override fun isReceiverAlive(): Boolean { - return senderReceiverService.isReceiverAlive() + val service = getActivity().getService() + if (service == null) { + return false + } + + return service.isReceiverAlive() } - override fun startSender(ip: String) { - Log.d(LOG_TAG, "Try start Sender") - senderReceiverService.startSender(ip, null) + override fun startSender(settings: AndroidSenderSettings) { + Log.i(LOG_TAG, "Starting sender if needed") + + if (!projectionAcquired) { + Log.e(LOG_TAG, "startSender must be called between acquireProjection/releaseProjection") + throw FlutterError(BAD_SEQUENCE_CODE, BAD_SEQUENCE_TEXT) + } + + val service = getActivity().getService() + if (service == null) { + Log.e(LOG_TAG, "Lost connection to service") + throw FlutterError(NO_SERVICE_CODE, NO_SERVICE_TEXT) + } + + if (!service.hasProjection()) { + Log.e(LOG_TAG, "Lost acquired projection") + throw FlutterError(NO_PROJECTION_CODE, NO_PROJECTION_TEXT) + } + + if (settings.captureType == AndroidCaptureType.CAPTURE_MIC && + !getActivity().hasPermission(Manifest.permission.RECORD_AUDIO)) { + Log.e(LOG_TAG, "Microphone permission must be granted when using CAPTURE_MIC") + throw FlutterError(NO_PERMISSION_CODE, NO_PERMISSION_TEXT) + } + + service.startSender(settings) } override fun stopSender() { - Log.d(LOG_TAG, "Try stop Sender") - senderReceiverService.stopSender() + Log.i(LOG_TAG, "Stopping sender if needed") + + val service = getActivity().getService() + if (service == null) { + Log.d(LOG_TAG, "Lost connection to service") + return + } + + service.stopSender() } override fun isSenderAlive(): Boolean { - return senderReceiverService.isReceiverAlive() + val service = getActivity().getService() + if (service == null) { + return false + } + + return service.isSenderAlive() } } diff --git a/android/app/src/main/kotlin/org/rocstreaming/rocdroid/MainActivity.kt b/android/app/src/main/kotlin/org/rocstreaming/rocdroid/MainActivity.kt index f7aebef..2b0ab66 100644 --- a/android/app/src/main/kotlin/org/rocstreaming/rocdroid/MainActivity.kt +++ b/android/app/src/main/kotlin/org/rocstreaming/rocdroid/MainActivity.kt @@ -1,63 +1,249 @@ package org.rocstreaming.rocdroid -import AndroidConnector +import AndroidListener +import AndroidServiceError +import AndroidServiceEvent import android.content.ComponentName +import android.content.Context.MEDIA_PROJECTION_SERVICE import android.content.Intent import android.content.ServiceConnection +import android.content.pm.PackageManager import android.media.AudioManager +import android.media.projection.MediaProjection +import android.media.projection.MediaProjectionManager import android.os.Bundle import android.os.IBinder import android.util.Log -import io.flutter.embedding.android.FlutterActivity +import androidx.activity.ComponentActivity +import androidx.activity.result.ActivityResult +import androidx.activity.result.ActivityResultLauncher +import androidx.activity.result.contract.ActivityResultContracts +import androidx.appcompat.app.AlertDialog +import androidx.core.content.ContextCompat +import io.flutter.embedding.android.FlutterFragmentActivity import io.flutter.embedding.engine.FlutterEngine -import org.rocstreaming.connector.AndroidConnectorImpl -import org.rocstreaming.service.SenderReceiverService +import org.rocstreaming.rocdroid.AndroidConnectorImpl +import org.rocstreaming.rocdroid.StreamingEventListener +import org.rocstreaming.rocdroid.StreamingService -private const val LOG_TAG = "[rocdroid.MainActivity]" +private const val LOG_TAG = "rocdroid.MainActivity" -class MainActivity: FlutterActivity() { - private lateinit var senderReceiverService: SenderReceiverService - +class MainActivity: FlutterFragmentActivity() { + // main activity is a singleton used by AndroidConnectorImpl + companion object { + lateinit var instance : MainActivity + } + + // non-null once successfully connected to server + // may temporarily become null when connection is lost + private var service: StreamingService? = null + + fun getService() : StreamingService? { + return service + } + + // called when we've started the service and connected to it + private var serviceStartedCallback: ((StreamingService) -> Unit)? = null + + // projectionRequestLauncher acquires access to projection from projectionManager + // and invokes projectionRequestCallback + private var projectionRequestCallback: ((MediaProjection?) -> Unit)? = null + private lateinit var projectionRequestLauncher: ActivityResultLauncher + private lateinit var projectionManager: MediaProjectionManager + + // permissionRequestLauncher invokes permissionRequestCallback when + // permission is granted or rejected + private var permissionRequestCallback: ((Boolean) -> Unit)? = null + private lateinit var permissionRequestLauncher: ActivityResultLauncher + + // bridge to invoke dart methods from kotlin + private lateinit var eventListener: AndroidListener + + // called at start override fun configureFlutterEngine(flutterEngine: FlutterEngine) { + Log.d(LOG_TAG, "Configuring flutter engine") + super.configureFlutterEngine(flutterEngine) + // AndroidConnector interface is generated by pigeon and is called from dart + // AndroidConnectorImpl implements its methods + // here we link them together AndroidConnector.setUp(flutterEngine.dartExecutor.binaryMessenger, AndroidConnectorImpl()) + + // AndroidListener class is generated by pigeon and allows kotlin to + // invoke dart methods + eventListener = AndroidListener(flutterEngine.dartExecutor.binaryMessenger) + } + + // when app is opened + override fun onCreate(savedInstanceState: Bundle?) { + Log.i(LOG_TAG, "Creating main activity") + + super.onCreate(savedInstanceState) + + instance = this + + // setup for permission requests + permissionRequestLauncher = registerForActivityResult( + ActivityResultContracts.RequestPermission(), this::onPermissionResult) + + // setup for projection request + projectionManager = getSystemService(MEDIA_PROJECTION_SERVICE) as MediaProjectionManager + projectionRequestLauncher = registerForActivityResult( + ActivityResultContracts.StartActivityForResult(), this::onProjectionResult) } - private val senderReceiverServiceConnection = object : ServiceConnection { + // when app is resumed + override fun onResume() { + Log.i(LOG_TAG, "Resuming main activity") + + super.onResume() + volumeControlStream = AudioManager.STREAM_MUSIC + } + + // when app is closed + override fun onDestroy() { + Log.i(LOG_TAG, "Destroying main activity") + + if (serviceConnection != null) { + unbindService(serviceConnection) + } + + super.onDestroy() + } + + // start service if not started yet + fun startService(callback: (StreamingService) -> Unit) { + if (service != null) { + Log.d(LOG_TAG, "Service already started, nothing to do") + callback(service!!) + return + } + + Log.d(LOG_TAG, "Starting service") + + // callback will be invoked from onServiceConnected() + serviceStartedCallback = callback + + val serviceIntent = Intent(this, StreamingService::class.java) + startForegroundService(serviceIntent); + bindService(serviceIntent, serviceConnection, BIND_AUTO_CREATE) + } + + // handler for events produced by streaming service + private val streamingEventHandler: StreamingEventListener = object : StreamingEventListener { + override fun onEvent(event: AndroidServiceEvent) { + Log.d(LOG_TAG, "Sending event: " + event.toString()) + runOnUiThread { + eventListener.onEvent(event) { result -> } + } + } + + override fun onError(error: AndroidServiceError) { + Log.d(LOG_TAG, "Sending error: " + error.toString()) + runOnUiThread { + eventListener.onError(error) { result -> } + } + } + } + + // handler for connect & disconnect events + private val serviceConnection = object : ServiceConnection { + // called when we've successfully connected to the service override fun onServiceConnected(componentName: ComponentName, binder: IBinder) { - senderReceiverService = (binder as SenderReceiverService.LocalBinder).getService() - - Log.d(LOG_TAG, "On service connected") + Log.i(LOG_TAG, "Service connected") + + // remember service reference + service = (binder as StreamingService.LocalBinder).getService() + service?.setEventListener(streamingEventHandler) - AndroidConnectorImpl.senderReceiverService = senderReceiverService + // for startService() + serviceStartedCallback?.invoke(service!!) + serviceStartedCallback = null } + // called when we've lost connectio to service override fun onServiceDisconnected(componentName: ComponentName) { + Log.w(LOG_TAG, "Service disconnected") - Log.d(LOG_TAG, "On service disconnected") + // forget service reference + service?.removeEventListener() + service = null - senderReceiverService.removeListeners() + // (re)start & reconnect + Log.d(LOG_TAG, "Initiating asynchronous reconnect") + val serviceIntent = Intent(this@MainActivity, StreamingService::class.java) + startForegroundService(serviceIntent); + bindService(serviceIntent, this, BIND_AUTO_CREATE) } } - override fun onCreate(savedInstanceState: Bundle?) { - super.onCreate(savedInstanceState) + fun requestProjection(callback: (MediaProjection?) -> Unit) { + Log.d(LOG_TAG, "Issuing media projection request") - Log.d(LOG_TAG, "Create Main Activity") + // callback will be invoked from onProjectionResult() + val projectionIntent = projectionManager.createScreenCaptureIntent() + projectionRequestCallback = callback + projectionRequestLauncher.launch(projectionIntent) + } - val serviceIntent = Intent(this, SenderReceiverService::class.java) - bindService(serviceIntent, senderReceiverServiceConnection, BIND_AUTO_CREATE) + private fun onProjectionResult(result: ActivityResult) { + if (result.data != null) { + Log.d(LOG_TAG, "Media projection acquired with code " + result.resultCode.toString()); + val projection = projectionManager.getMediaProjection(result.resultCode, result.data!!) + projectionRequestCallback?.invoke(projection) + projectionRequestCallback = null + } else { + Log.d(LOG_TAG, "Media projection rejected with code " + result.resultCode.toString()); + projectionRequestCallback?.invoke(null) + projectionRequestCallback = null + } } - override fun onResume() { - super.onResume() - volumeControlStream = AudioManager.STREAM_MUSIC + fun hasPermission(permission: String): Boolean { + return ContextCompat.checkSelfPermission(this, permission) == PackageManager.PERMISSION_GRANTED } - override fun onDestroy() { - super.onDestroy() - unbindService(senderReceiverServiceConnection) + fun requestPermission(permission: String, titleID: Int, messageID: Int, callback: (Boolean) -> Unit) { + if (ContextCompat.checkSelfPermission(this, permission) == PackageManager.PERMISSION_GRANTED) { + Log.d(LOG_TAG, "Permission already granted, nothing to do") + callback(true) + return + } + + if (shouldShowRequestPermissionRationale(permission)) { + Log.d(LOG_TAG, "Showing permission dialog and waiting for response") + AlertDialog.Builder(this).apply { + setTitle(titleID) + setMessage(messageID) + setPositiveButton(R.string.ok) { _, _ -> + Log.d(LOG_TAG, "User approved permission, issuing request") + // callback will be invoked from onPermissionResult() + permissionRequestCallback = callback + permissionRequestLauncher.launch(permission) + } + setNegativeButton(R.string.cancel) { dialog, _ -> + Log.w(LOG_TAG, "User declined permission, rejecting") + dialog.dismiss() + callback(false) + } + }.show() + } else { + Log.d(LOG_TAG, "Requesting permission directly") + // callback will be invoked from onPermissionResult() + permissionRequestCallback = callback + permissionRequestLauncher.launch(permission) + } + } + + private fun onPermissionResult(isGranted: Boolean) { + if (isGranted) { + Log.d(LOG_TAG, "Permission is granted"); + } else { + Log.w(LOG_TAG, "Permission is rejected"); + } + permissionRequestCallback?.invoke(isGranted) + permissionRequestCallback = null } } diff --git a/android/app/src/main/kotlin/org/rocstreaming/service/SenderReceiverService.kt b/android/app/src/main/kotlin/org/rocstreaming/service/SenderReceiverService.kt index b8ffec8..7ac3358 100644 --- a/android/app/src/main/kotlin/org/rocstreaming/service/SenderReceiverService.kt +++ b/android/app/src/main/kotlin/org/rocstreaming/service/SenderReceiverService.kt @@ -1,5 +1,9 @@ -package org.rocstreaming.service +package org.rocstreaming.rocdroid +import AndroidReceiverSettings +import AndroidSenderSettings +import AndroidServiceError +import AndroidServiceEvent import android.app.Notification import android.app.NotificationChannel import android.app.NotificationManager @@ -18,11 +22,8 @@ import android.media.AudioTrack import android.media.MediaRecorder import android.media.projection.MediaProjection import android.os.Binder -import android.os.Build import android.os.IBinder import android.util.Log -import androidx.annotation.RequiresApi -import androidx.appcompat.app.AlertDialog import org.rocstreaming.rocdroid.MainActivity import org.rocstreaming.rocdroid.R import org.rocstreaming.roctoolkit.ChannelSet @@ -41,31 +42,45 @@ import org.rocstreaming.roctoolkit.Slot private const val SAMPLE_RATE = 44100 private const val BUFFER_SIZE = 100 -private const val DEFAULT_RTP_PORT_SOURCE = 10001 -private const val DEFAULT_RTP_PORT_REPAIR = 10002 - -private const val CHANNEL_ID = "SenderReceiverService" +private const val NOTIFICATION_CHANNEL_ID = "StreamingService" private const val NOTIFICATION_ID = 1 -private const val BROADCAST_STOP_SENDER_ACTION = - "org.rocstreaming.rocdroid.NotificationSenderStopAction" -private const val BROADCAST_STOP_RECEIVER_ACTION = - "org.rocstreaming.rocdroid.NotificationReceiverStopAction" +private const val NOTIFICATION_ACTION_DELETE = + "org.rocstreaming.rocdroid.NotificationActionDelete" +private const val NOTIFICATION_ACTION_STOP_SENDER = + "org.rocstreaming.rocdroid.NotificationActionStopSender" +private const val NOTIFICATION_ACTION_STOP_RECEIVER = + "org.rocstreaming.rocdroid.NotificationActionStopReceiver" + +private const val LOG_TAG = "rocdroid.StreamingService" -private const val LOG_TAG = "[rocdroid.SenderReceiverService]" +// Used to report asynchronous events and errors from service. +interface StreamingEventListener { + fun onEvent(event: AndroidServiceEvent) + fun onError(error: AndroidServiceError) +} -class SenderReceiverService : Service() { +// This service runs even when the app is closed. +// Related docs: +// https://medium.com/@domen.lanisnik/guide-to-foreground-services-on-android-9d0127dc8f9a +// https://developer.android.com/reference/android/media/projection/MediaProjectionManager +class StreamingService : Service() { private var receiverThread: Thread? = null private var senderThread: Thread? = null - private var receiverChanged: ((Boolean) -> Unit)? = null - private var senderChanged: ((Boolean) -> Unit)? = null - private var isForegroundRunning = false + private var receiverStarted = false + private var senderStarted = false + private var eventListener: StreamingEventListener? = null + private var autoDetach: Boolean = true + private var currentProjection: MediaProjection? = null - private val notificationStopActionReceiver: BroadcastReceiver = object : BroadcastReceiver() { + private val notificationActionHandler: BroadcastReceiver = object : BroadcastReceiver() { override fun onReceive(context: Context, intent: Intent) { + Log.i(LOG_TAG, "Handling notification action: " + intent.action) + when (intent.action) { - BROADCAST_STOP_SENDER_ACTION -> stopSender() - BROADCAST_STOP_RECEIVER_ACTION -> stopReceiver() + NOTIFICATION_ACTION_DELETE -> stopAllNoNotification() + NOTIFICATION_ACTION_STOP_SENDER -> stopSender() + NOTIFICATION_ACTION_STOP_RECEIVER -> stopReceiver() } } } @@ -73,244 +88,392 @@ class SenderReceiverService : Service() { private val binder = LocalBinder() inner class LocalBinder : Binder() { - fun getService(): SenderReceiverService = this@SenderReceiverService + fun getService(): StreamingService = this@StreamingService } override fun onBind(intent: Intent): IBinder { - Log.d(LOG_TAG, "Bind Service") + Log.i(LOG_TAG, "Binding service") return binder } override fun onCreate() { - Log.d(LOG_TAG, "Creating Sender/Receiver Service") + Log.i(LOG_TAG, "Creating service") - createNotificationChannel() - registerReceiver( - notificationStopActionReceiver, - IntentFilter().apply { - addAction(BROADCAST_STOP_SENDER_ACTION) - addAction(BROADCAST_STOP_RECEIVER_ACTION) - } - ) + super.onCreate() + + initNotifications() + } + + override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int { + Log.i(LOG_TAG, "Starting service") + + startForeground(NOTIFICATION_ID, buildNotification()) + + return START_STICKY } override fun onDestroy() { - Log.d(LOG_TAG, "Destroying Sender/Receiver Service") + Log.i(LOG_TAG, "Destroying service") + + terminate() + deinitNotifications() super.onDestroy() - unregisterReceiver(notificationStopActionReceiver) } - private fun createNotificationChannel() { - Log.d(LOG_TAG, "Creating Notification Channel") + private fun terminate() { + Log.d(LOG_TAG, "Stopping threads") - val channel = NotificationChannel( - CHANNEL_ID, - getString(R.string.notification_channel_name), - NotificationManager.IMPORTANCE_LOW - ) - val service = getSystemService(Context.NOTIFICATION_SERVICE) as NotificationManager - service.createNotificationChannel(channel) - } - - private fun buildNotification(sending: Boolean, receiving: Boolean): Notification { - Log.d( - LOG_TAG, - String.format( - "Building Notification for %s %s", - if (sending) "Sender" else "", - if (receiving) "Receiver" else "" - ) - ) + stopSender() + stopReceiver() - val mainActivityIntent = Intent(this, MainActivity::class.java) - val pendingMainActivityIntent = PendingIntent.getActivity( - this, - 0, - mainActivityIntent, - PendingIntent.FLAG_UPDATE_CURRENT - ) - val stopSenderIntent = Intent(BROADCAST_STOP_SENDER_ACTION) - val pendingStopSenderIntent = PendingIntent.getBroadcast( - this@SenderReceiverService, - 0, - stopSenderIntent, - PendingIntent.FLAG_UPDATE_CURRENT - ) - val stopReceiverIntent = Intent(BROADCAST_STOP_RECEIVER_ACTION) - val pendingStopReceiverIntent = PendingIntent.getBroadcast( - this@SenderReceiverService, - 0, - stopReceiverIntent, - PendingIntent.FLAG_UPDATE_CURRENT - ) - val stopSenderAction = Notification.Action.Builder( - Icon.createWithResource(this@SenderReceiverService, R.drawable.ic_stop), - getString(R.string.notification_stop_sender_action), - pendingStopSenderIntent - ).build() - val stopReceiverAction = Notification.Action.Builder( - Icon.createWithResource(this@SenderReceiverService, R.drawable.ic_stop), - getString(R.string.notification_stop_receiver_action), - pendingStopReceiverIntent - ).build() - return Notification.Builder(this, CHANNEL_ID).apply { - setContentTitle(getString(R.string.notification_title)) - setContentText(getContentText(sending, receiving)) - setSmallIcon(R.drawable.ic_notification) - setVisibility(Notification.VISIBILITY_PUBLIC) - setContentIntent(pendingMainActivityIntent) - if (sending) { - addAction(stopSenderAction) - } - if (receiving) { - addAction(stopReceiverAction) - } - }.build() + Log.d(LOG_TAG, "Waiting threads") + + senderThread?.join() + receiverThread?.join() + + Log.d(LOG_TAG, "Stopping media projection") + + currentProjection?.stop() + currentProjection = null + + Log.d(LOG_TAG, "Stopping service") + + stopForeground(true) } - private fun updateNotification(sending: Boolean, receiving: Boolean) { - Log.d( - LOG_TAG, - String.format( - "Updating Notification for %s %s", - if (sending) "Sender" else "", - if (receiving) "Receiver" else "" - ) - ) + @Synchronized + fun hasProjection(): Boolean { + return currentProjection != null + } - if (!isForegroundRunning) { - return - } + @Synchronized + fun attachProjection(projection: MediaProjection) { + Log.i(LOG_TAG, "Attaching media projection") + + currentProjection = projection + } + + @Synchronized + fun enableAutoDetach() { + autoDetach = true + autoDetachProjection() + } + + @Synchronized + fun disableAutoDetach() { + autoDetach = false + } + + private fun autoDetachProjection() { + if (!senderStarted && !receiverStarted && currentProjection != null && autoDetach) { + Log.i(LOG_TAG, "Detaching media projection") - if (receiving || sending) { - val notification = buildNotification(sending, receiving) - val notificationManager = - getSystemService(Context.NOTIFICATION_SERVICE) as NotificationManager - notificationManager.notify(NOTIFICATION_ID, notification) - } else { - stopForegroundService() + currentProjection?.stop() + currentProjection = null } } - private fun getContentText(sending: Boolean, receiving: Boolean): String { - Log.d(LOG_TAG, "Getting Notification Content Text") + @Synchronized + fun isSenderAlive(): Boolean { + return senderStarted + } + + @Synchronized + fun startSender(settings: AndroidSenderSettings) { + if (senderStarted) return + + Log.i(LOG_TAG, "Starting sender") - if (sending && receiving) { - return getString(R.string.notification_sender_and_receiver_running) + val projection = currentProjection + if (projection == null) { + throw IllegalStateException("Projection not attached") } - if (receiving) { - return getString(R.string.notification_receiver_running) + + val previousThread = senderThread + + senderThread = Thread { + try { + if (previousThread != null) { + Log.d(LOG_TAG, "Joining previois sender thread") + previousThread.join() + } + + runSenderThread(settings, projection) + } finally { + val currentThread = Thread.currentThread() + + synchronized(this@StreamingService) { + if (senderThread == currentThread) { + stopSender() + } else { + Log.d(LOG_TAG, "Ignoring dangling sender thread") + } + } + } } - if (sending) { - return getString(R.string.notification_sender_running) + + senderStarted = true + senderThread!!.start() + + updateNotification() + reportEvent(AndroidServiceEvent.SENDER_STATE_CHANGED) + } + + @Synchronized + fun stopSender() { + if (!senderStarted) return + + Log.i(LOG_TAG, "Stopping sender") + + senderStarted = false + senderThread?.interrupt() + + updateNotification() + autoDetachProjection() + + reportEvent(AndroidServiceEvent.SENDER_STATE_CHANGED) + } + + @Synchronized + fun isReceiverAlive(): Boolean { + return receiverStarted + } + + @Synchronized + fun startReceiver(settings: AndroidReceiverSettings) { + if (receiverStarted) return + + Log.i(LOG_TAG, "Starting receiver") + + val projection = currentProjection + if (projection == null) { + throw IllegalStateException("Projection not attached") } - return getString(R.string.notification_sender_and_receiver_not_running) // this shouldn't happen + + val previousThread = receiverThread + + receiverThread = Thread { + try { + if (previousThread != null) { + Log.d(LOG_TAG, "Joining previois receiver thread") + previousThread.join() + } + + runReceiverThread(settings, projection) + } finally { + val currentThread = Thread.currentThread() + + synchronized(this@StreamingService) { + if (receiverThread == currentThread) { + stopReceiver() + } else { + Log.d(LOG_TAG, "Ignoring dangling receiver thread") + } + } + } + } + + receiverStarted = true + receiverThread!!.start() + + updateNotification() + reportEvent(AndroidServiceEvent.RECEIVER_STATE_CHANGED) + } + + @Synchronized + fun stopReceiver() { + if (!receiverStarted) return + + Log.i(LOG_TAG, "Stopping receiver") + + receiverStarted = false + receiverThread?.interrupt() + + updateNotification() + autoDetachProjection() + + reportEvent(AndroidServiceEvent.RECEIVER_STATE_CHANGED) } - fun preStartSender() { - Log.d(LOG_TAG, "Prestart Sender") + @Synchronized + fun stopAllNoNotification() { + if (!senderStarted && !receiverStarted) return + + if (senderStarted) { + Log.i(LOG_TAG, "Stopping sender") - if (isForegroundRunning) { - updateNotification(true, isReceiverAlive()) - } else { - startForegroundService(true, isReceiverAlive()) + senderStarted = false + senderThread?.interrupt() + + reportEvent(AndroidServiceEvent.SENDER_STATE_CHANGED) + } + + if (receiverStarted) { + Log.i(LOG_TAG, "Stopping receiver") + + receiverStarted = false + receiverThread?.interrupt() + + reportEvent(AndroidServiceEvent.RECEIVER_STATE_CHANGED) } + + autoDetachProjection() } - fun startSender(ip: String, projection: MediaProjection?) { - Log.d(LOG_TAG, "Starting Sender") + @Synchronized + fun setEventListener(listener: StreamingEventListener) { + Log.d(LOG_TAG, "Setting event listener") - if (senderThread?.isAlive == true) return + eventListener = listener + } - senderThread = Thread { - val record = createAudioRecord(projection) + @Synchronized + fun removeEventListener() { + Log.d(LOG_TAG, "Removing event listener") + + eventListener = null + } + + @Synchronized + private fun reportEvent(event: AndroidServiceEvent) { + Log.d(LOG_TAG, "Reporting event: " + event.toString()) + + eventListener?.onEvent(event) + } + + @Synchronized + private fun reportError(error: AndroidServiceError) { + Log.d(LOG_TAG, "Reporting error: " + error.toString()) + + eventListener?.onError(error) + } - val config = RocSenderConfig.builder() + private fun runSenderThread(settings: AndroidSenderSettings, projection: MediaProjection) { + Log.d(LOG_TAG, "Running sender thread") + + var audioRecord: AudioRecord? = null + + try { + try { + if (settings.captureType == AndroidCaptureType.CAPTURE_APPS) { + audioRecord = createProjectionAudioRecord(projection) + } else { + audioRecord = createMicrophoneAudioRecord() + } + } catch (e: Exception) { + Log.e(LOG_TAG, "Failed to create audio record: " + e.toString()) + reportError(AndroidServiceError.AUDIO_RECORD_FAILED) + return + } + + if (audioRecord.state != AudioRecord.STATE_INITIALIZED) { + Log.e(LOG_TAG, "Failed to initialize audio record: " + audioRecord.state.toString()) + reportError(AndroidServiceError.AUDIO_RECORD_FAILED) + return + } + + val senderConfig = RocSenderConfig.builder() .frameSampleRate(44100) .frameChannels(ChannelSet.STEREO) .frameEncoding(FrameEncoding.PCM_FLOAT) - .clockSource(ClockSource.INTERNAL) + .clockSource(ClockSource.EXTERNAL) .build() RocContext().use { context -> - if (record.state != AudioRecord.STATE_INITIALIZED) return@use - - record.startRecording() - - RocSender(context, config).use useSender@{ sender -> - + RocSender(context, senderConfig).use useSender@{ sender -> try { sender.connect( Slot.DEFAULT, Interface.AUDIO_SOURCE, - Endpoint(Protocol.RTP_RS8M_SOURCE, ip, DEFAULT_RTP_PORT_SOURCE) + Endpoint(Protocol.RTP_RS8M_SOURCE, + settings.host, + settings.sourcePort.toInt()) ) sender.connect( Slot.DEFAULT, Interface.AUDIO_REPAIR, - Endpoint(Protocol.RS8M_REPAIR, ip, DEFAULT_RTP_PORT_REPAIR) + Endpoint(Protocol.RS8M_REPAIR, + settings.host, + settings.repairPort.toInt()) ) } catch (e: Exception) { - AlertDialog.Builder(this@SenderReceiverService).apply { - setTitle(getString(R.string.invalid_ip_title)) - setMessage(getString(R.string.invalid_ip_message)) - setCancelable(false) - setPositiveButton(R.string.ok) { _, _ -> } - }.show() + Log.e(LOG_TAG, "Failed to connect sender: " + e.toString()) + reportError(AndroidServiceError.SENDER_CONNECT_FAILED) return@useSender } - senderChanged?.invoke(true) + audioRecord.startRecording() val samples = FloatArray(BUFFER_SIZE) while (!Thread.currentThread().isInterrupted) { - record.read(samples, 0, samples.size, AudioRecord.READ_BLOCKING) + audioRecord.read(samples, 0, samples.size, AudioRecord.READ_BLOCKING) sender.write(samples) } } - - record.stop() - record.release() - senderChanged?.invoke(false) - updateNotification(false, isReceiverAlive()) } - } + } finally { + Log.d(LOG_TAG, "Releasing sender resources") - senderThread!!.start() + audioRecord?.stop() + audioRecord?.release() + + Log.d(LOG_TAG, "Exiting sender thread") + } } - fun startReceiver() { - Log.d(LOG_TAG, "Starting Receiver") + private fun runReceiverThread(settings: AndroidReceiverSettings, projection: MediaProjection) { + Log.d(LOG_TAG, "Running receiver thread") - if (receiverThread?.isAlive == true) return + var audioTrack: AudioTrack? = null - receiverThread = Thread { - val audioTrack = createAudioTrack() - audioTrack.play() + try { + try { + audioTrack = createAudioTrack() + } catch (e: Exception) { + Log.e(LOG_TAG, "Failed to create audio track: " + e.toString()) + reportError(AndroidServiceError.AUDIO_TRACK_FAILED) + return + } - val config = RocReceiverConfig.builder() + if (audioTrack.state != AudioTrack.STATE_INITIALIZED) { + Log.e(LOG_TAG, "Failed to initialize audio track: " + audioTrack.state.toString()) + reportError(AndroidServiceError.AUDIO_TRACK_FAILED) + return + } + + val receiverConfig = RocReceiverConfig.builder() .frameSampleRate(44100) .frameChannels(ChannelSet.STEREO) .frameEncoding(FrameEncoding.PCM_FLOAT) - .clockSource(ClockSource.INTERNAL) + .clockSource(ClockSource.EXTERNAL) .build() RocContext().use { context -> - RocReceiver(context, config).use { receiver -> - receiver.bind( - Slot.DEFAULT, - Interface.AUDIO_SOURCE, - Endpoint(Protocol.RTP_RS8M_SOURCE, "0.0.0.0", DEFAULT_RTP_PORT_SOURCE) - ) - receiver.bind( - Slot.DEFAULT, - Interface.AUDIO_REPAIR, - Endpoint(Protocol.RS8M_REPAIR, "0.0.0.0", DEFAULT_RTP_PORT_REPAIR) - ) - - receiverChanged?.invoke(true) + RocReceiver(context, receiverConfig).use useReceiver@{ receiver -> + try { + receiver.bind( + Slot.DEFAULT, + Interface.AUDIO_SOURCE, + Endpoint(Protocol.RTP_RS8M_SOURCE, + "0.0.0.0", + settings.sourcePort.toInt()) + ) + receiver.bind( + Slot.DEFAULT, + Interface.AUDIO_REPAIR, + Endpoint(Protocol.RS8M_REPAIR, + "0.0.0.0", + settings.repairPort.toInt()) + ) + } catch (e: Exception) { + Log.e(LOG_TAG, "Failed to bind receiver: " + e.toString()) + reportError(AndroidServiceError.RECEIVER_BIND_FAILED) + return@useReceiver + } + + audioTrack.play() val samples = FloatArray(BUFFER_SIZE) while (!Thread.currentThread().isInterrupted) { @@ -320,67 +483,18 @@ class SenderReceiverService : Service() { } } - audioTrack.release() - receiverChanged?.invoke(false) - updateNotification(isSenderAlive(), false) - } + } finally { + Log.d(LOG_TAG, "Releasing receiver resources") - receiverThread!!.start() + audioTrack?.stop() + audioTrack?.release() - if (isForegroundRunning) { - updateNotification(isSenderAlive(), true) - } else { - startForegroundService(isSenderAlive(), true) + Log.d(LOG_TAG, "Exiting receiver thread") } } - fun stopSender() { - Log.d(LOG_TAG, "Stopping Sender") - - senderThread?.interrupt() - } - - fun stopReceiver() { - Log.d(LOG_TAG, "Stopping Receiver") - - receiverThread?.interrupt() - } - - fun isReceiverAlive(): Boolean { - Log.d(LOG_TAG, "Checking If Receiver Alive") - - return receiverThread?.isAlive == true - } - - fun isSenderAlive(): Boolean { - Log.d(LOG_TAG, "Checking If Sender Alive") - - return senderThread?.isAlive == true - } - - private fun startForegroundService(sending: Boolean, receiving: Boolean) { - Log.d( - LOG_TAG, - String.format( - "Starting Foreground Service for %s %s", - if (sending) "Sender" else "", - if (receiving) "Receiver" else "" - ) - ) - - isForegroundRunning = true - startForeground(NOTIFICATION_ID, buildNotification(sending, receiving)) - } - - private fun stopForegroundService() { - Log.d(LOG_TAG, "Stopping Foreground Service") - - isForegroundRunning = false - stopForeground(true) - } - private fun createAudioTrack(): AudioTrack { - Log.d(LOG_TAG, "Creating Audio Track") + Log.d(LOG_TAG, "Creating audio track") val audioAttributes = AudioAttributes.Builder().apply { setUsage(AudioAttributes.USAGE_MEDIA) @@ -405,8 +519,8 @@ class SenderReceiverService : Service() { }.build() } - private fun createAudioRecord(projection: MediaProjection?): AudioRecord { - Log.d(LOG_TAG, "Creating Audio Record") + private fun createMicrophoneAudioRecord(): AudioRecord { + Log.d(LOG_TAG, "Creating microphone audio record") val format = AudioFormat.Builder().apply { setSampleRate(SAMPLE_RATE) @@ -418,32 +532,31 @@ class SenderReceiverService : Service() { AudioFormat.CHANNEL_IN_STEREO, AudioFormat.ENCODING_PCM_FLOAT ) - - return if (projection != null && Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { - createPaybackRecord(projection, format, bufferSize) - } else { - AudioRecord.Builder().apply { - setAudioSource(MediaRecorder.AudioSource.DEFAULT) - setAudioFormat(format) - setBufferSizeInBytes(bufferSize) - }.build() - } + return AudioRecord.Builder().apply { + setAudioSource(MediaRecorder.AudioSource.DEFAULT) + setAudioFormat(format) + setBufferSizeInBytes(bufferSize) + }.build() } - @RequiresApi(Build.VERSION_CODES.Q) - private fun createPaybackRecord( - projection: MediaProjection, - format: AudioFormat, - bufferSize: Int - ): AudioRecord { - Log.d(LOG_TAG, "Creating Playback Record") + private fun createProjectionAudioRecord(projection: MediaProjection): AudioRecord { + Log.d(LOG_TAG, "Creating projection audio record") + val format = AudioFormat.Builder().apply { + setSampleRate(SAMPLE_RATE) + setChannelMask(AudioFormat.CHANNEL_IN_STEREO) + setEncoding(AudioFormat.ENCODING_PCM_FLOAT) + }.build() + val bufferSize = AudioRecord.getMinBufferSize( + SAMPLE_RATE, + AudioFormat.CHANNEL_IN_STEREO, + AudioFormat.ENCODING_PCM_FLOAT + ) val config = AudioPlaybackCaptureConfiguration.Builder(projection).apply { addMatchingUsage(AudioAttributes.USAGE_MEDIA) addMatchingUsage(AudioAttributes.USAGE_UNKNOWN) addMatchingUsage(AudioAttributes.USAGE_GAME) }.build() - return AudioRecord.Builder().apply { setAudioPlaybackCaptureConfig(config) setAudioFormat(format) @@ -451,26 +564,137 @@ class SenderReceiverService : Service() { }.build() } - fun setSenderStateChangedListeners( - senderChanged: (Boolean) -> Unit - ) { - Log.d(LOG_TAG, "Setting Sender State Changed Listener") + private fun initNotifications() { + Log.d(LOG_TAG, "Initializing notifications") - this.senderChanged = senderChanged + val channel = NotificationChannel( + NOTIFICATION_CHANNEL_ID, + getString(R.string.notification_channel_name), + NotificationManager.IMPORTANCE_LOW + ) + + val notificationManager = + getSystemService(Context.NOTIFICATION_SERVICE) as NotificationManager + + notificationManager.createNotificationChannel(channel) + + registerReceiver( + notificationActionHandler, + IntentFilter().apply { + addAction(NOTIFICATION_ACTION_DELETE) + addAction(NOTIFICATION_ACTION_STOP_SENDER) + addAction(NOTIFICATION_ACTION_STOP_RECEIVER) + }, + RECEIVER_EXPORTED + ) } - fun setReceiverStateChangedListeners( - receiverChanged: (Boolean) -> Unit - ) { - Log.d(LOG_TAG, "Setting Receiver State Changed Listener") + private fun deinitNotifications() { + Log.d(LOG_TAG, "Deinitializing notifications") - this.receiverChanged = receiverChanged + unregisterReceiver(notificationActionHandler) } - fun removeListeners() { - Log.d(LOG_TAG, "Removing State Changed Listeners") + private fun buildNotification(): Notification { + Log.d(LOG_TAG, "Building notification: actions=" + getNotificationDesc()) + + // invoked when notification is tapped + // we want to open main activity + val contentIntent = Intent(this, MainActivity::class.java) + val pendingContentIntent = PendingIntent.getActivity( + this, + 0, + contentIntent, + PendingIntent.FLAG_IMMUTABLE + ) + + // invoked when notification is dismissed (swiped away) + // we want to stop sender & receiver + val deleteIntent = Intent(NOTIFICATION_ACTION_DELETE) + val pendingDeleteIntent = PendingIntent.getBroadcast( + this, + 0, + deleteIntent, + PendingIntent.FLAG_IMMUTABLE + ) - this.receiverChanged = null - this.senderChanged = null + // invoked when "stop sender" notification button is pressed + val stopSenderIntent = Intent(NOTIFICATION_ACTION_STOP_SENDER) + val pendingStopSenderIntent = PendingIntent.getBroadcast( + this, + 0, + stopSenderIntent, + PendingIntent.FLAG_IMMUTABLE + ) + val stopSenderAction = Notification.Action.Builder( + Icon.createWithResource(this@StreamingService, R.drawable.ic_stop), + getString(R.string.notification_stop_sender_action), + pendingStopSenderIntent + ).build() + + // invoked when "stop receiver" notification button is pressed + val stopReceiverIntent = Intent(NOTIFICATION_ACTION_STOP_RECEIVER) + val pendingStopReceiverIntent = PendingIntent.getBroadcast( + this, + 0, + stopReceiverIntent, + PendingIntent.FLAG_IMMUTABLE + ) + val stopReceiverAction = Notification.Action.Builder( + Icon.createWithResource(this@StreamingService, R.drawable.ic_stop), + getString(R.string.notification_stop_receiver_action), + pendingStopReceiverIntent + ).build() + + return Notification.Builder(this, NOTIFICATION_CHANNEL_ID).apply { + // appearance + setSmallIcon(R.drawable.ic_notification) + setContentTitle(getString(R.string.notification_title)) + setContentText(getNotificationText()) + // when notification is tapped + setContentIntent(pendingContentIntent) + // when notification is swiped away + setDeleteIntent(pendingDeleteIntent); + // don't allow to dimiss notification on lock screen + setOngoing(true) + // show on lock screen + setVisibility(Notification.VISIBILITY_PUBLIC) + // notification buttons + if (senderStarted) { + addAction(stopSenderAction) + } + if (receiverStarted) { + addAction(stopReceiverAction) + } + }.build() + } + + private fun updateNotification() { + Log.d(LOG_TAG, "Updating notification: actions=" + getNotificationDesc()) + + val notification = buildNotification() + val notificationManager = + getSystemService(Context.NOTIFICATION_SERVICE) as NotificationManager + + notificationManager.notify(NOTIFICATION_ID, notification) + } + + private fun getNotificationText(): String { + return when { + senderStarted && receiverStarted -> + getString(R.string.notification_sender_and_receiver_running) + senderStarted -> getString(R.string.notification_sender_running) + receiverStarted -> getString(R.string.notification_receiver_running) + else -> getString(R.string.notification_sender_and_receiver_not_running) + } + } + + private fun getNotificationDesc(): String { + return when { + senderStarted && receiverStarted -> "[sender, receiver]" + senderStarted -> "[sender]" + receiverStarted -> "[receiver]" + else -> "[]" + } } } diff --git a/android/app/src/main/res/values/strings.xml b/android/app/src/main/res/values/strings.xml index 63d4c2f..9df2271 100644 --- a/android/app/src/main/res/values/strings.xml +++ b/android/app/src/main/res/values/strings.xml @@ -1,22 +1,14 @@ Roc Droid - Receiver - Start receiver - Stop receiver + OK + Cancel + + Notification permission + The app needs permission to post notifications. - Sender - Receiver IP - 192.168.0.1 - Start sender - Stop sender - Invalid IP address - The receiver IP address looks invalid. Mic permission The app needs permission to record audio from your phone mic. - Thanks! The sender will now start. - - OK Sender and Receiver foreground service Receiver running @@ -26,33 +18,4 @@ Roc State Stop Sender Stop Receiver - - 1. Start sender on the remote device - 2. Use one of IP addresses of this device as the remote on the sender - 5. Start receiver on this device - - 1. Start receiver on the remote device - 4. Put IP address of the remote receiver device below - 5. Choose source to capture audio from - Currently playing apps - Microphone - 6. Start sender on this device - Source to capture audio from - - %s. Use this port for source stream - %s. Use this port for repair stream - About - Source Code - Bug Tracker - Contributors - Mozilla Public License 2.0 - - https://github.com/roc-streaming/roc-droid - https://github.com/roc-streaming/roc-droid/issues - https://github.com/roc-streaming/roc-droid/graphs/contributors - https://github.com/roc-streaming/roc-droid/blob/main/LICENSE - App Logo - Copy - Open dropdown - Roc Droid receiver diff --git a/dodo.py b/dodo.py index 6f8932d..dc96ea2 100644 --- a/dodo.py +++ b/dodo.py @@ -1,17 +1,18 @@ -from doit.tools import title_with_actions, LongRunning, Interactive from doit import get_var +from doit.tools import title_with_actions, LongRunning, Interactive import atexit import glob import os import platform import shutil import signal +import subprocess import sys atexit.register( lambda: shutil.rmtree('__pycache__', ignore_errors=True)) if os.name == 'posix': - signal.signal(signal.SIGINT, lambda s, f: os._exit(0)) + signal.signal(signal.SIGINT, lambda s, f: exit(1)) DOIT_CONFIG = { 'default_tasks': ['analyze', 'test'], @@ -51,8 +52,17 @@ def task_test(): # doit launch [variant=debug|release] def task_launch(): """deploy and run on device""" + def do_cleanup(): + subprocess.call( + 'adb shell am force-stop org.rocstreaming.rocdroid', + shell=True) + def register_cleanup(): + atexit.register(do_cleanup) return { - 'actions': [Interactive(f'flutter run --{VARIANT}')], + 'actions': [ + register_cleanup, + Interactive(f'flutter run --{VARIANT}'), + ], 'title': title_with_actions, } @@ -112,7 +122,7 @@ def task_gen_agent(): """run flutter pigeon Agent code generation""" return { 'basename': 'gen:agent', - 'actions': ['dart run pigeon --input lib/src/agent/android_connector.dart'], + 'actions': ['dart run pigeon --input lib/src/agent/android_bridge.decl.dart'], 'title': title_with_actions, } diff --git a/l10n.yaml b/l10n.yaml index 20f5476..da77ba3 100644 --- a/l10n.yaml +++ b/l10n.yaml @@ -1,3 +1,3 @@ arb-dir: lib/src/ui/localization template-arb-file: app_en.arb -output-localization-file: app_localizations.dart \ No newline at end of file +output-localization-file: app_localizations.dart diff --git a/lib/src/agent.dart b/lib/src/agent.dart index c136d93..f7d4feb 100644 --- a/lib/src/agent.dart +++ b/lib/src/agent.dart @@ -1,2 +1,4 @@ /// Agent export definition. +export 'agent/android_backend.dart'; +export 'agent/android_bridge.g.dart'; export 'agent/backend.dart'; diff --git a/lib/src/agent/android_backend.dart b/lib/src/agent/android_backend.dart index 7147f66..8a124b3 100644 --- a/lib/src/agent/android_backend.dart +++ b/lib/src/agent/android_backend.dart @@ -1,37 +1,164 @@ -import 'android_connector.g.dart'; +import 'package:logger/logger.dart'; + +import 'android_bridge.g.dart'; import 'backend.dart'; -/// ???. -class AndroidBackend implements Backend { - final AndroidConnector _connector = AndroidConnector(); +/// Android-specific implementation of Backend interface. +/// +/// Uses AndroidConnector, which is a bridge to AndroidConnectorImpl, +/// which is implemented in Kotlin. +/// +/// Implements AndroidListener, which is invoked from kotlin. +class AndroidBackend implements Backend, AndroidListener { + final Logger _logger; + final AndroidConnector _connector; + + AndroidBackend({required Logger logger}) + : _logger = logger, + _connector = AndroidConnector() { + // Tell kotlin that we implement AndroidListener interface, so + // that it would invoke our methods. + AndroidListener.setUp(this); + } + /// Inherited from Backend interface. + /// Invoked from model. @override - Future startReceiver() async { - await _connector.startReceiver(); + Future> getLocalAddresses() async { + // Cast List to List. + return (await _connector.getLocalAddresses()) + .where((addr) => addr != null) + .cast() + .toList(); } + /// Inherited from Backend interface. + /// Invoked from model. + @override + Future startReceiver(AndroidReceiverSettings settings) async { + if (await _connector.isReceiverAlive()) { + _logger.d("Receiver already started"); + return; + } + + _logger.i("Starting receiver"); + + // Ensure service can post notifications. + if (!await _connector.requestNotifications()) { + return; + } + + try { + // First request media projection if not granted yet and acquire it + // while we're starting receiver. + if (!await _connector.acquireProjection()) { + return; + } + + // Then start receiver. + await _connector.startReceiver(settings); + } finally { + // Then release projection, i.e. allow service to stop it when it's + // not needed anymore. + await _connector.releaseProjection(); + } + } + + /// Inherited from Backend interface. + /// Invoked from model. @override Future stopReceiver() async { + if (!await _connector.isReceiverAlive()) { + _logger.d("Receiver already stopped"); + return; + } + + _logger.i("Stopping receiver"); + await _connector.stopReceiver(); } + /// Inherited from Backend interface. + /// Invoked from model. @override Future isReceiverAlive() async { return await _connector.isReceiverAlive(); } + /// Inherited from Backend interface. + /// Invoked from model. @override - Future startSender(String ip) async { - await _connector.startSender(ip); + Future startSender(AndroidSenderSettings settings) async { + if (await _connector.isSenderAlive()) { + _logger.d("Sender already started"); + return; + } + + _logger.i("Starting sender"); + + // Ensure service can post notifications. + if (!await _connector.requestNotifications()) { + return; + } + + // If user want's to capture from microphone, we need to request + // permission before starting the sender. + if (settings.captureType == AndroidCaptureType.captureMic) { + if (!await _connector.requestMicrophone()) { + return; + } + } + + try { + // First request media projection if not granted yet and acquire it + // while we're starting sender. + if (!await _connector.acquireProjection()) { + return; + } + + // Then start sender. + await _connector.startSender(settings); + } finally { + // Then release projection, i.e. allow service to stop it when it's + // not needed anymore. + await _connector.releaseProjection(); + } } + /// Inherited from Backend interface. + /// Invoked from model. @override Future stopSender() async { + if (!await _connector.isSenderAlive()) { + _logger.d("Sender already stopped"); + return; + } + + _logger.i("Stopping sender"); + await _connector.stopSender(); } + /// Inherited from Backend interface. + /// Invoked from model. @override Future isSenderAlive() async { return await _connector.isSenderAlive(); } + + /// Inherited from AndroidListener interface. + /// Invoked from kotlin. + @override + void onEvent(AndroidServiceEvent eventCode) async { + // TODO: notify model about state change + _logger.d("Got async event: $eventCode"); + } + + /// Inherited from AndroidListener interface. + /// Invoked from kotlin. + @override + void onError(AndroidServiceError errorCode) { + // TODO: display error to user (e.g. using toast) + _logger.d("Got async error: $errorCode"); + } } diff --git a/lib/src/agent/android_connector.dart b/lib/src/agent/android_connector.dart index 5b4d4d3..3453d3e 100644 --- a/lib/src/agent/android_connector.dart +++ b/lib/src/agent/android_connector.dart @@ -1,32 +1,159 @@ import 'package:pigeon/pigeon.dart'; +/// This file (android_connector.decl.dart) is not actually included into build and +/// is never imported by other dart code. It's only used during code generation to +/// produce two other files: +/// - android_connector.g.dart +/// - AndroidConnector.g.kt @ConfigurePigeon(PigeonOptions( - dartOut: 'lib/src/agent/android_connector.g.dart', + dartOut: 'lib/src/agent/android_bridge.g.dart', dartOptions: DartOptions(), kotlinOut: - 'android/app/src/main/kotlin/org/rocstreaming/connector/AndroidConnector.g.kt', + 'android/app/src/main/java/org/rocstreaming/rocdroid/AndroidBridge.g.kt', kotlinOptions: KotlinOptions(), dartPackageName: 'roc_droid', )) -/// ???. +/// Receiver settings. +class AndroidReceiverSettings { + AndroidReceiverSettings({ + required this.sourcePort, + required this.repairPort, + }); + + /// Local port to receive source packets. + final int sourcePort; + + /// Local port to receive repair packets. + final int repairPort; +} + +/// Sender settings. +class AndroidSenderSettings { + AndroidSenderSettings({ + required this.captureType, + required this.host, + required this.sourcePort, + required this.repairPort, + }); + + /// From where to capture stream. + final AndroidCaptureType captureType; + + /// IP address or hostname where to send packets. + final String host; + + /// Remote port where to send source packets. + final int sourcePort; + + /// Remote port where to send repair packets. + final int repairPort; +} + +/// Where sender gets sound. +enum AndroidCaptureType { + /// Capture from locally playing apps. + captureApps, + + /// Capture from local microphone. + captureMic, +} + +/// Allows to invoke kotlin methods from dart. +/// +/// This declaration emits 2 classes: +/// dart: AndroidConnector implementation class, which methods invoke kotlin +/// methods under the hood (via platform channels) +/// kotlin: AndroidConnector interface, which we implement in +/// AndroidConnectorImpl, where the actual work is done @HostApi() abstract class AndroidConnector { - void startReceiver(); + /// Get list of IP addresses of available network interfaces. + List getLocalAddresses(); + + /// Request permission to post notifications, if no already granted. + /// Must be called before acquiring projection first time. + /// If returns false, user rejected permission and notifications won't appear. + @async + bool requestNotifications(); + + /// Request permission to capture local microphone, if not already granted. + /// Must be called before starting sender when using AndroidCaptureType.captureMic. + /// If returns false, user rejected permission and sender won't start. + @async + bool requestMicrophone(); + + /// Request access to media projection, if not already granted. + /// Must be called before starting sender or receiver. + /// If returns false, user rejected access and sender/receiver won't start. + /// Throws exception if: + /// - lost connection to foreground service + @async + bool acquireProjection(); + + /// Allow service to stop projection when it's not needed. + /// Must be called after *starting* sender or receiver. + void releaseProjection(); + + /// Start receiver. + /// Receiver gets stream from network and plays to local speakers. + /// Must be called between acquireProjection() and releaseProjection(). + /// Throws exception if: + /// - lost connection to foreground service + /// - media projection wasn't acquired + void startReceiver(AndroidReceiverSettings settings); + /// Stop receiver. void stopReceiver(); + /// Check if receiver is running. bool isReceiverAlive(); - void startSender(String ip); + /// Start sender. + /// Sender gets stream from local microphone OR media system apps, and streams to network. + /// Must be called between acquireProjection() and releaseProjection(). + /// Throws exception if: + /// - lost connection to foreground service + /// - media projection not acquired + /// - microphone permission is needed and wasn't granted + void startSender(AndroidSenderSettings settings); + /// Stop sender. void stopSender(); + /// Check if sender is running. bool isSenderAlive(); } -// /// ???. -// @FlutterApi() -// abstract class FlutterHandler { -// void textChanged(String text); -// } +/// Asynchronous events produces by android service. +enum AndroidServiceEvent { + senderStateChanged, + receiverStateChanged, +} + +/// Asynchronous errors produces by android service. +enum AndroidServiceError { + audioRecordFailed, + audioTrackFailed, + senderConnectFailed, + receiverBindFailed, +} + +/// Allows to invoke dart methods from kotlin. +/// +/// This declaration emits 2 classes: +/// dart: AndroidListener interface class, which is implemented +/// by AndroidBackend +/// kotlin: AndroidListener implementation class, which methods invoke +/// dart methods under the hood (via platform channels) +@FlutterApi() +abstract class AndroidListener { + /// Invoked when an asynchronous event occurs. + /// For example, sender is started or stopped by UI, notification button, + /// tile button, or because of failure. + void onEvent(AndroidServiceEvent eventCode); + + /// Invoked when an asynchronous error occurs. + /// For example, sender encounters network error. + void onError(AndroidServiceError errorCode); +} diff --git a/lib/src/agent/android_connector.g.dart b/lib/src/agent/android_connector.g.dart index b79bdff..7c9ba01 100644 --- a/lib/src/agent/android_connector.g.dart +++ b/lib/src/agent/android_connector.g.dart @@ -15,12 +15,165 @@ PlatformException _createConnectionError(String channelName) { ); } +List wrapResponse({Object? result, PlatformException? error, bool empty = false}) { + if (empty) { + return []; + } + if (error == null) { + return [result]; + } + return [error.code, error.message, error.details]; +} + +/// Where sender gets sound. +enum AndroidCaptureType { + /// Capture from locally playing apps. + captureApps, + /// Capture from local microphone. + captureMic, +} + +/// Asynchronous events produces by android service. +enum AndroidServiceEvent { + senderStateChanged, + receiverStateChanged, +} + +/// Asynchronous errors produces by android service. +enum AndroidServiceError { + audioRecordFailed, + audioTrackFailed, + senderConnectFailed, + receiverBindFailed, +} + +/// Receiver settings. +class AndroidReceiverSettings { + AndroidReceiverSettings({ + required this.sourcePort, + required this.repairPort, + }); + + /// Local port to receive source packets. + int sourcePort; + + /// Local port to receive repair packets. + int repairPort; + + Object encode() { + return [ + sourcePort, + repairPort, + ]; + } + + static AndroidReceiverSettings decode(Object result) { + result as List; + return AndroidReceiverSettings( + sourcePort: result[0]! as int, + repairPort: result[1]! as int, + ); + } +} + +/// Sender settings. +class AndroidSenderSettings { + AndroidSenderSettings({ + required this.captureType, + required this.host, + required this.sourcePort, + required this.repairPort, + }); + + /// From where to capture stream. + AndroidCaptureType captureType; + + /// IP address or hostname where to send packets. + String host; + + /// Remote port where to send source packets. + int sourcePort; + + /// Remote port where to send repair packets. + int repairPort; + + Object encode() { + return [ + captureType, + host, + sourcePort, + repairPort, + ]; + } + + static AndroidSenderSettings decode(Object result) { + result as List; + return AndroidSenderSettings( + captureType: result[0]! as AndroidCaptureType, + host: result[1]! as String, + sourcePort: result[2]! as int, + repairPort: result[3]! as int, + ); + } +} + class _PigeonCodec extends StandardMessageCodec { const _PigeonCodec(); + @override + void writeValue(WriteBuffer buffer, Object? value) { + if (value is int) { + buffer.putUint8(4); + buffer.putInt64(value); + } else if (value is AndroidCaptureType) { + buffer.putUint8(129); + writeValue(buffer, value.index); + } else if (value is AndroidServiceEvent) { + buffer.putUint8(130); + writeValue(buffer, value.index); + } else if (value is AndroidServiceError) { + buffer.putUint8(131); + writeValue(buffer, value.index); + } else if (value is AndroidReceiverSettings) { + buffer.putUint8(132); + writeValue(buffer, value.encode()); + } else if (value is AndroidSenderSettings) { + buffer.putUint8(133); + writeValue(buffer, value.encode()); + } else { + super.writeValue(buffer, value); + } + } + + @override + Object? readValueOfType(int type, ReadBuffer buffer) { + switch (type) { + case 129: + final int? value = readValue(buffer) as int?; + return value == null ? null : AndroidCaptureType.values[value]; + case 130: + final int? value = readValue(buffer) as int?; + return value == null ? null : AndroidServiceEvent.values[value]; + case 131: + final int? value = readValue(buffer) as int?; + return value == null ? null : AndroidServiceError.values[value]; + case 132: + return AndroidReceiverSettings.decode(readValue(buffer)!); + case 133: + return AndroidSenderSettings.decode(readValue(buffer)!); + default: + return super.readValueOfType(type, buffer); + } + } } -/// ???. +/// Allows to invoke kotlin methods from dart. +/// +/// This declaration emits 2 classes: +/// dart: AndroidConnector implementation class, which methods invoke kotlin +/// methods under the hood (via platform channels) +/// kotlin: AndroidConnector interface, which we implement in +/// AndroidConnectorImpl, where the actual work is done class AndroidConnector { /// Constructor for [AndroidConnector]. The [binaryMessenger] named argument is /// available for dependency injection. If it is left null, the default @@ -34,8 +187,130 @@ class AndroidConnector { final String pigeonVar_messageChannelSuffix; - Future startReceiver() async { - final String pigeonVar_channelName = 'dev.flutter.pigeon.roc_droid.AndroidConnector.startReceiver$pigeonVar_messageChannelSuffix'; + /// Get list of IP addresses of available network interfaces. + Future> getLocalAddresses() async { + final String pigeonVar_channelName = 'dev.flutter.pigeon.roc_droid.AndroidConnector.getLocalAddresses$pigeonVar_messageChannelSuffix'; + final BasicMessageChannel pigeonVar_channel = BasicMessageChannel( + pigeonVar_channelName, + pigeonChannelCodec, + binaryMessenger: pigeonVar_binaryMessenger, + ); + final List? pigeonVar_replyList = + await pigeonVar_channel.send(null) as List?; + if (pigeonVar_replyList == null) { + throw _createConnectionError(pigeonVar_channelName); + } else if (pigeonVar_replyList.length > 1) { + throw PlatformException( + code: pigeonVar_replyList[0]! as String, + message: pigeonVar_replyList[1] as String?, + details: pigeonVar_replyList[2], + ); + } else if (pigeonVar_replyList[0] == null) { + throw PlatformException( + code: 'null-error', + message: 'Host platform returned null value for non-null return value.', + ); + } else { + return (pigeonVar_replyList[0] as List?)!.cast(); + } + } + + /// Request permission to post notifications, if no already granted. + /// Must be called before acquiring projection first time. + /// If returns false, user rejected permission and notifications won't appear. + Future requestNotifications() async { + final String pigeonVar_channelName = 'dev.flutter.pigeon.roc_droid.AndroidConnector.requestNotifications$pigeonVar_messageChannelSuffix'; + final BasicMessageChannel pigeonVar_channel = BasicMessageChannel( + pigeonVar_channelName, + pigeonChannelCodec, + binaryMessenger: pigeonVar_binaryMessenger, + ); + final List? pigeonVar_replyList = + await pigeonVar_channel.send(null) as List?; + if (pigeonVar_replyList == null) { + throw _createConnectionError(pigeonVar_channelName); + } else if (pigeonVar_replyList.length > 1) { + throw PlatformException( + code: pigeonVar_replyList[0]! as String, + message: pigeonVar_replyList[1] as String?, + details: pigeonVar_replyList[2], + ); + } else if (pigeonVar_replyList[0] == null) { + throw PlatformException( + code: 'null-error', + message: 'Host platform returned null value for non-null return value.', + ); + } else { + return (pigeonVar_replyList[0] as bool?)!; + } + } + + /// Request permission to capture local microphone, if not already granted. + /// Must be called before starting sender when using AndroidCaptureType.captureMic. + /// If returns false, user rejected permission and sender won't start. + Future requestMicrophone() async { + final String pigeonVar_channelName = 'dev.flutter.pigeon.roc_droid.AndroidConnector.requestMicrophone$pigeonVar_messageChannelSuffix'; + final BasicMessageChannel pigeonVar_channel = BasicMessageChannel( + pigeonVar_channelName, + pigeonChannelCodec, + binaryMessenger: pigeonVar_binaryMessenger, + ); + final List? pigeonVar_replyList = + await pigeonVar_channel.send(null) as List?; + if (pigeonVar_replyList == null) { + throw _createConnectionError(pigeonVar_channelName); + } else if (pigeonVar_replyList.length > 1) { + throw PlatformException( + code: pigeonVar_replyList[0]! as String, + message: pigeonVar_replyList[1] as String?, + details: pigeonVar_replyList[2], + ); + } else if (pigeonVar_replyList[0] == null) { + throw PlatformException( + code: 'null-error', + message: 'Host platform returned null value for non-null return value.', + ); + } else { + return (pigeonVar_replyList[0] as bool?)!; + } + } + + /// Request access to media projection, if not already granted. + /// Must be called before starting sender or receiver. + /// If returns false, user rejected access and sender/receiver won't start. + /// Throws exception if: + /// - lost connection to foreground service + Future acquireProjection() async { + final String pigeonVar_channelName = 'dev.flutter.pigeon.roc_droid.AndroidConnector.acquireProjection$pigeonVar_messageChannelSuffix'; + final BasicMessageChannel pigeonVar_channel = BasicMessageChannel( + pigeonVar_channelName, + pigeonChannelCodec, + binaryMessenger: pigeonVar_binaryMessenger, + ); + final List? pigeonVar_replyList = + await pigeonVar_channel.send(null) as List?; + if (pigeonVar_replyList == null) { + throw _createConnectionError(pigeonVar_channelName); + } else if (pigeonVar_replyList.length > 1) { + throw PlatformException( + code: pigeonVar_replyList[0]! as String, + message: pigeonVar_replyList[1] as String?, + details: pigeonVar_replyList[2], + ); + } else if (pigeonVar_replyList[0] == null) { + throw PlatformException( + code: 'null-error', + message: 'Host platform returned null value for non-null return value.', + ); + } else { + return (pigeonVar_replyList[0] as bool?)!; + } + } + + /// Allow service to stop projection when it's not needed. + /// Must be called after *starting* sender or receiver. + Future releaseProjection() async { + final String pigeonVar_channelName = 'dev.flutter.pigeon.roc_droid.AndroidConnector.releaseProjection$pigeonVar_messageChannelSuffix'; final BasicMessageChannel pigeonVar_channel = BasicMessageChannel( pigeonVar_channelName, pigeonChannelCodec, @@ -56,6 +331,35 @@ class AndroidConnector { } } + /// Start receiver. + /// Receiver gets stream from network and plays to local speakers. + /// Must be called between acquireProjection() and releaseProjection(). + /// Throws exception if: + /// - lost connection to foreground service + /// - media projection wasn't acquired + Future startReceiver(AndroidReceiverSettings settings) async { + final String pigeonVar_channelName = 'dev.flutter.pigeon.roc_droid.AndroidConnector.startReceiver$pigeonVar_messageChannelSuffix'; + final BasicMessageChannel pigeonVar_channel = BasicMessageChannel( + pigeonVar_channelName, + pigeonChannelCodec, + binaryMessenger: pigeonVar_binaryMessenger, + ); + final List? pigeonVar_replyList = + await pigeonVar_channel.send([settings]) as List?; + if (pigeonVar_replyList == null) { + throw _createConnectionError(pigeonVar_channelName); + } else if (pigeonVar_replyList.length > 1) { + throw PlatformException( + code: pigeonVar_replyList[0]! as String, + message: pigeonVar_replyList[1] as String?, + details: pigeonVar_replyList[2], + ); + } else { + return; + } + } + + /// Stop receiver. Future stopReceiver() async { final String pigeonVar_channelName = 'dev.flutter.pigeon.roc_droid.AndroidConnector.stopReceiver$pigeonVar_messageChannelSuffix'; final BasicMessageChannel pigeonVar_channel = BasicMessageChannel( @@ -78,6 +382,7 @@ class AndroidConnector { } } + /// Check if receiver is running. Future isReceiverAlive() async { final String pigeonVar_channelName = 'dev.flutter.pigeon.roc_droid.AndroidConnector.isReceiverAlive$pigeonVar_messageChannelSuffix'; final BasicMessageChannel pigeonVar_channel = BasicMessageChannel( @@ -105,7 +410,14 @@ class AndroidConnector { } } - Future startSender(String ip) async { + /// Start sender. + /// Sender gets stream from local microphone OR media system apps, and streams to network. + /// Must be called between acquireProjection() and releaseProjection(). + /// Throws exception if: + /// - lost connection to foreground service + /// - media projection not acquired + /// - microphone permission is needed and wasn't granted + Future startSender(AndroidSenderSettings settings) async { final String pigeonVar_channelName = 'dev.flutter.pigeon.roc_droid.AndroidConnector.startSender$pigeonVar_messageChannelSuffix'; final BasicMessageChannel pigeonVar_channel = BasicMessageChannel( pigeonVar_channelName, @@ -113,7 +425,7 @@ class AndroidConnector { binaryMessenger: pigeonVar_binaryMessenger, ); final List? pigeonVar_replyList = - await pigeonVar_channel.send([ip]) as List?; + await pigeonVar_channel.send([settings]) as List?; if (pigeonVar_replyList == null) { throw _createConnectionError(pigeonVar_channelName); } else if (pigeonVar_replyList.length > 1) { @@ -127,6 +439,7 @@ class AndroidConnector { } } + /// Stop sender. Future stopSender() async { final String pigeonVar_channelName = 'dev.flutter.pigeon.roc_droid.AndroidConnector.stopSender$pigeonVar_messageChannelSuffix'; final BasicMessageChannel pigeonVar_channel = BasicMessageChannel( @@ -149,6 +462,7 @@ class AndroidConnector { } } + /// Check if sender is running. Future isSenderAlive() async { final String pigeonVar_channelName = 'dev.flutter.pigeon.roc_droid.AndroidConnector.isSenderAlive$pigeonVar_messageChannelSuffix'; final BasicMessageChannel pigeonVar_channel = BasicMessageChannel( @@ -176,3 +490,77 @@ class AndroidConnector { } } } + +/// Allows to invoke dart methods from kotlin. +/// +/// This declaration emits 2 classes: +/// dart: AndroidListener interface class, which is implemented +/// by AndroidBackend +/// kotlin: AndroidListener implementation class, which methods invoke +/// dart methods under the hood (via platform channels) +abstract class AndroidListener { + static const MessageCodec pigeonChannelCodec = _PigeonCodec(); + + /// Invoked when an asynchronous event occurs. + /// For example, sender is started or stopped by UI, notification button, + /// tile button, or because of failure. + void onEvent(AndroidServiceEvent eventCode); + + /// Invoked when an asynchronous error occurs. + /// For example, sender encounters network error. + void onError(AndroidServiceError errorCode); + + static void setUp(AndroidListener? api, {BinaryMessenger? binaryMessenger, String messageChannelSuffix = '',}) { + messageChannelSuffix = messageChannelSuffix.isNotEmpty ? '.$messageChannelSuffix' : ''; + { + final BasicMessageChannel pigeonVar_channel = BasicMessageChannel( + 'dev.flutter.pigeon.roc_droid.AndroidListener.onEvent$messageChannelSuffix', pigeonChannelCodec, + binaryMessenger: binaryMessenger); + if (api == null) { + pigeonVar_channel.setMessageHandler(null); + } else { + pigeonVar_channel.setMessageHandler((Object? message) async { + assert(message != null, + 'Argument for dev.flutter.pigeon.roc_droid.AndroidListener.onEvent was null.'); + final List args = (message as List?)!; + final AndroidServiceEvent? arg_eventCode = (args[0] as AndroidServiceEvent?); + assert(arg_eventCode != null, + 'Argument for dev.flutter.pigeon.roc_droid.AndroidListener.onEvent was null, expected non-null AndroidServiceEvent.'); + try { + api.onEvent(arg_eventCode!); + return wrapResponse(empty: true); + } on PlatformException catch (e) { + return wrapResponse(error: e); + } catch (e) { + return wrapResponse(error: PlatformException(code: 'error', message: e.toString())); + } + }); + } + } + { + final BasicMessageChannel pigeonVar_channel = BasicMessageChannel( + 'dev.flutter.pigeon.roc_droid.AndroidListener.onError$messageChannelSuffix', pigeonChannelCodec, + binaryMessenger: binaryMessenger); + if (api == null) { + pigeonVar_channel.setMessageHandler(null); + } else { + pigeonVar_channel.setMessageHandler((Object? message) async { + assert(message != null, + 'Argument for dev.flutter.pigeon.roc_droid.AndroidListener.onError was null.'); + final List args = (message as List?)!; + final AndroidServiceError? arg_errorCode = (args[0] as AndroidServiceError?); + assert(arg_errorCode != null, + 'Argument for dev.flutter.pigeon.roc_droid.AndroidListener.onError was null, expected non-null AndroidServiceError.'); + try { + api.onError(arg_errorCode!); + return wrapResponse(empty: true); + } on PlatformException catch (e) { + return wrapResponse(error: e); + } catch (e) { + return wrapResponse(error: PlatformException(code: 'error', message: e.toString())); + } + }); + } + } + } +} diff --git a/lib/src/agent/backend.dart b/lib/src/agent/backend.dart index 8dec27d..48e0760 100644 --- a/lib/src/agent/backend.dart +++ b/lib/src/agent/backend.dart @@ -1,14 +1,32 @@ -/// ???. +import 'android_bridge.g.dart'; + +// This is intended to be platform-independent interface for streaming backend, +// implemented differently on mobile and desktop. +// +// Currently we support only android and for now this class just mirrors +// AndroidConnector interface mostly one-to-one. +// +// Later we will add support for desktop (via rocd), and rework this interface +// to be more generic. In particular, we'll remove start/stop sender/receiver +// methods, and introduce concepts of Hosts, Devices, and Links. +// +// Sender and receiver will become a special case of creating a Link between local +// Device and remote Host. +// +// AndroidConnector, used by AndroidBackend, likely will remain unmodified. But +// AndroidBackend will be reworked to implement new generic interface on top of +// simpler AndroidConnector. +// +// Similarly, AndroidReceiverSettings and AndroidSenderSettings likely will remain +// as is, but Backend will use some higher-level classes (e.g. LinkConfig, HostConfig). abstract class Backend { - Future startReceiver(); + Future> getLocalAddresses(); + Future startReceiver(AndroidReceiverSettings settings); Future stopReceiver(); - Future isReceiverAlive(); - Future startSender(String ip); - + Future startSender(AndroidSenderSettings settings); Future stopSender(); - Future isSenderAlive(); } diff --git a/lib/src/model/model_root.dart b/lib/src/model/model_root.dart index 1400563..ca234b2 100644 --- a/lib/src/model/model_root.dart +++ b/lib/src/model/model_root.dart @@ -1,7 +1,6 @@ import 'package:logger/logger.dart'; -import '../agent/android_backend.dart'; -import '../agent/backend.dart'; +import '../agent.dart'; import 'receiver.dart'; import 'sender.dart'; @@ -16,7 +15,7 @@ class ModelRoot { // Temporary assignment of Android backend. // In the future we will probably need some mechanism // to decide which type of backend to assign. - Backend backend = AndroidBackend(); + Backend backend = AndroidBackend(logger: mainLogger); receiver = Receiver(mainLogger, backend); sender = Sender(mainLogger, backend); logger = mainLogger; diff --git a/lib/src/model/receiver.dart b/lib/src/model/receiver.dart index c4d8c59..33bfad0 100644 --- a/lib/src/model/receiver.dart +++ b/lib/src/model/receiver.dart @@ -3,7 +3,7 @@ import 'dart:collection'; import 'package:logger/logger.dart'; import 'package:mobx/mobx.dart'; -import '../agent/backend.dart'; +import '../agent.dart'; part 'receiver.g.dart'; @@ -56,7 +56,10 @@ abstract class _Receiver with Store { } // Main backend call - await _backend.startReceiver(); + await _backend.startReceiver(AndroidReceiverSettings( + sourcePort: 10001, + repairPort: 10002, + )); var status = await _backend.isReceiverAlive(); _logger.i('Trying to start the receiver. roc service status: $status'); diff --git a/lib/src/model/sender.dart b/lib/src/model/sender.dart index bfa2e75..6d2f1ad 100644 --- a/lib/src/model/sender.dart +++ b/lib/src/model/sender.dart @@ -1,7 +1,7 @@ import 'package:logger/logger.dart'; import 'package:mobx/mobx.dart'; -import '../agent/backend.dart'; +import '../agent.dart'; import 'capture_source_type.dart'; part 'sender.g.dart'; @@ -62,7 +62,12 @@ abstract class _Sender with Store { } // Main backend call - await _backend.startSender(receiverIP); + await _backend.startSender(AndroidSenderSettings( + captureType: AndroidCaptureType.captureApps, + host: receiverIP, + sourcePort: 10001, + repairPort: 10002, + )); var status = await _backend.isSenderAlive(); _logger.i('Trying to start the sender. roc service status: $status'); @@ -91,27 +96,23 @@ abstract class _Sender with Store { @action void setSourcePort(int value) { _sourcePort = value; - _logger.d('Sender source port value changed to: ${_sourcePort}'); } // Update repair port value. @action void setRepairPort(int value) { _repairPort = value; - _logger.d('Sender repair port value changed to: ${_repairPort}'); } // Update the active source port. @action void setReceiverIP(String value) { _receiverIP = value; - _logger.d('Sender active source port value changed to: ${_receiverIP}'); } // Update the active the user-selected capture source enum. @action void setCaptureSource(CaptureSourceType value) { _captureSource = value; - _logger.d('Capture source enum value changed to: ${_captureSource}'); } }