From 5e40ee0965d59adbabffba5f0cf78cb14f3d69f3 Mon Sep 17 00:00:00 2001 From: crc-32 Date: Fri, 18 Oct 2024 15:03:58 +0100 Subject: [PATCH] dictation audio handling --- .../kotlin/io/rebble/cobble/MainActivity.kt | 3 + .../12.json | 523 ++++++++++++++++++ .../rebble/cobble/shared/di/koin.android.kt | 1 + .../voice/NullDictationService.android.kt | 30 + .../io/rebble/cobble/shared/di/VoiceModule.kt | 11 + .../shared/domain/voice/DictationService.kt | 2 +- .../domain/voice/DictationServiceResponse.kt | 10 +- .../domain/voice/NullDictationService.kt | 34 ++ .../shared/handlers/AudioStreamHandler.kt | 2 +- .../shared/handlers/VoiceSessionHandler.kt | 51 +- .../domain/voice/NullDictationService.ios.kt | 5 + 11 files changed, 647 insertions(+), 25 deletions(-) create mode 100644 android/shared/schemas/io.rebble.cobble.shared.database.AppDatabase/12.json create mode 100644 android/shared/src/androidMain/kotlin/io/rebble/cobble/shared/domain/voice/NullDictationService.android.kt create mode 100644 android/shared/src/commonMain/kotlin/io/rebble/cobble/shared/di/VoiceModule.kt create mode 100644 android/shared/src/commonMain/kotlin/io/rebble/cobble/shared/domain/voice/NullDictationService.kt create mode 100644 android/shared/src/iosMain/kotlin/io/rebble/cobble/shared/domain/voice/NullDictationService.ios.kt diff --git a/android/app/src/main/kotlin/io/rebble/cobble/MainActivity.kt b/android/app/src/main/kotlin/io/rebble/cobble/MainActivity.kt index e323f9fc..65f20e5e 100644 --- a/android/app/src/main/kotlin/io/rebble/cobble/MainActivity.kt +++ b/android/app/src/main/kotlin/io/rebble/cobble/MainActivity.kt @@ -1,6 +1,9 @@ package io.rebble.cobble import android.content.Intent +import android.media.MediaCodec +import android.media.MediaCodecList +import android.media.MediaFormat import android.net.Uri import android.os.Bundle import androidx.activity.compose.setContent diff --git a/android/shared/schemas/io.rebble.cobble.shared.database.AppDatabase/12.json b/android/shared/schemas/io.rebble.cobble.shared.database.AppDatabase/12.json new file mode 100644 index 00000000..9cb88da5 --- /dev/null +++ b/android/shared/schemas/io.rebble.cobble.shared.database.AppDatabase/12.json @@ -0,0 +1,523 @@ +{ + "formatVersion": 1, + "database": { + "version": 12, + "identityHash": "0f71b8464e1fb6d1e3cd0f6754b90233", + "entities": [ + { + "tableName": "Calendar", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `platformId` TEXT NOT NULL, `name` TEXT NOT NULL, `ownerName` TEXT NOT NULL, `ownerId` TEXT NOT NULL, `color` INTEGER NOT NULL, `enabled` INTEGER NOT NULL)", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "platformId", + "columnName": "platformId", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "name", + "columnName": "name", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "ownerName", + "columnName": "ownerName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "ownerId", + "columnName": "ownerId", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "color", + "columnName": "color", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "enabled", + "columnName": "enabled", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": true, + "columnNames": [ + "id" + ] + }, + "indices": [ + { + "name": "index_Calendar_platformId", + "unique": true, + "columnNames": [ + "platformId" + ], + "orders": [], + "createSql": "CREATE UNIQUE INDEX IF NOT EXISTS `index_Calendar_platformId` ON `${TABLE_NAME}` (`platformId`)" + } + ] + }, + { + "tableName": "TimelinePin", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`itemId` TEXT NOT NULL, `parentId` TEXT NOT NULL, `backingId` TEXT, `timestamp` INTEGER NOT NULL, `duration` INTEGER, `type` TEXT NOT NULL, `isVisible` INTEGER NOT NULL, `isFloating` INTEGER NOT NULL, `isAllDay` INTEGER NOT NULL, `persistQuickView` INTEGER NOT NULL, `layout` TEXT NOT NULL, `attributesJson` TEXT, `actionsJson` TEXT, `nextSyncAction` TEXT, PRIMARY KEY(`itemId`))", + "fields": [ + { + "fieldPath": "itemId", + "columnName": "itemId", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "parentId", + "columnName": "parentId", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "backingId", + "columnName": "backingId", + "affinity": "TEXT" + }, + { + "fieldPath": "timestamp", + "columnName": "timestamp", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "duration", + "columnName": "duration", + "affinity": "INTEGER" + }, + { + "fieldPath": "type", + "columnName": "type", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "isVisible", + "columnName": "isVisible", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "isFloating", + "columnName": "isFloating", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "isAllDay", + "columnName": "isAllDay", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "persistQuickView", + "columnName": "persistQuickView", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "layout", + "columnName": "layout", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "attributesJson", + "columnName": "attributesJson", + "affinity": "TEXT" + }, + { + "fieldPath": "actionsJson", + "columnName": "actionsJson", + "affinity": "TEXT" + }, + { + "fieldPath": "nextSyncAction", + "columnName": "nextSyncAction", + "affinity": "TEXT" + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "itemId" + ] + } + }, + { + "tableName": "PersistedNotification", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`sbnKey` TEXT NOT NULL, `packageName` TEXT NOT NULL, `postTime` INTEGER NOT NULL, `title` TEXT NOT NULL, `text` TEXT NOT NULL, `groupKey` TEXT, PRIMARY KEY(`sbnKey`))", + "fields": [ + { + "fieldPath": "sbnKey", + "columnName": "sbnKey", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "packageName", + "columnName": "packageName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "postTime", + "columnName": "postTime", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "title", + "columnName": "title", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "text", + "columnName": "text", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "groupKey", + "columnName": "groupKey", + "affinity": "TEXT" + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "sbnKey" + ] + } + }, + { + "tableName": "CachedPackageInfo", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` TEXT NOT NULL, `name` TEXT NOT NULL, `flags` INTEGER NOT NULL, `updated` INTEGER NOT NULL, PRIMARY KEY(`id`))", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "name", + "columnName": "name", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "flags", + "columnName": "flags", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "updated", + "columnName": "updated", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "id" + ] + } + }, + { + "tableName": "NotificationChannel", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`packageId` TEXT NOT NULL, `channelId` TEXT NOT NULL, `name` TEXT, `description` TEXT, `conversationId` TEXT, `shouldNotify` INTEGER NOT NULL, PRIMARY KEY(`packageId`, `channelId`))", + "fields": [ + { + "fieldPath": "packageId", + "columnName": "packageId", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "channelId", + "columnName": "channelId", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "name", + "columnName": "name", + "affinity": "TEXT" + }, + { + "fieldPath": "description", + "columnName": "description", + "affinity": "TEXT" + }, + { + "fieldPath": "conversationId", + "columnName": "conversationId", + "affinity": "TEXT" + }, + { + "fieldPath": "shouldNotify", + "columnName": "shouldNotify", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "packageId", + "channelId" + ] + } + }, + { + "tableName": "SyncedLockerEntry", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` TEXT NOT NULL, `uuid` TEXT NOT NULL, `version` TEXT NOT NULL, `title` TEXT NOT NULL, `type` TEXT NOT NULL, `hearts` INTEGER NOT NULL, `developerName` TEXT NOT NULL, `developerId` TEXT, `configurable` INTEGER NOT NULL, `timelineEnabled` INTEGER NOT NULL, `removeLink` TEXT NOT NULL, `shareLink` TEXT NOT NULL, `pbwLink` TEXT NOT NULL, `pbwReleaseId` TEXT NOT NULL, `pbwIconResourceId` INTEGER NOT NULL DEFAULT 0, `nextSyncAction` TEXT NOT NULL, `order` INTEGER NOT NULL DEFAULT -1, `lastOpened` INTEGER, `local` INTEGER NOT NULL DEFAULT 0, PRIMARY KEY(`id`))", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "uuid", + "columnName": "uuid", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "version", + "columnName": "version", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "title", + "columnName": "title", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "type", + "columnName": "type", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "hearts", + "columnName": "hearts", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "developerName", + "columnName": "developerName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "developerId", + "columnName": "developerId", + "affinity": "TEXT" + }, + { + "fieldPath": "configurable", + "columnName": "configurable", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "timelineEnabled", + "columnName": "timelineEnabled", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "removeLink", + "columnName": "removeLink", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "shareLink", + "columnName": "shareLink", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "pbwLink", + "columnName": "pbwLink", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "pbwReleaseId", + "columnName": "pbwReleaseId", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "pbwIconResourceId", + "columnName": "pbwIconResourceId", + "affinity": "INTEGER", + "notNull": true, + "defaultValue": "0" + }, + { + "fieldPath": "nextSyncAction", + "columnName": "nextSyncAction", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "order", + "columnName": "order", + "affinity": "INTEGER", + "notNull": true, + "defaultValue": "-1" + }, + { + "fieldPath": "lastOpened", + "columnName": "lastOpened", + "affinity": "INTEGER" + }, + { + "fieldPath": "local", + "columnName": "local", + "affinity": "INTEGER", + "notNull": true, + "defaultValue": "0" + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "id" + ] + }, + "indices": [ + { + "name": "index_SyncedLockerEntry_uuid", + "unique": true, + "columnNames": [ + "uuid" + ], + "orders": [], + "createSql": "CREATE UNIQUE INDEX IF NOT EXISTS `index_SyncedLockerEntry_uuid` ON `${TABLE_NAME}` (`uuid`)" + } + ] + }, + { + "tableName": "SyncedLockerEntryPlatform", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`platformEntryId` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `lockerEntryId` TEXT NOT NULL, `sdkVersion` TEXT NOT NULL, `processInfoFlags` INTEGER NOT NULL, `name` TEXT NOT NULL, `description` TEXT NOT NULL, `icon` TEXT, `list` TEXT, `screenshot` TEXT, FOREIGN KEY(`lockerEntryId`) REFERENCES `SyncedLockerEntry`(`id`) ON UPDATE NO ACTION ON DELETE CASCADE )", + "fields": [ + { + "fieldPath": "platformEntryId", + "columnName": "platformEntryId", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "lockerEntryId", + "columnName": "lockerEntryId", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "sdkVersion", + "columnName": "sdkVersion", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "processInfoFlags", + "columnName": "processInfoFlags", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "name", + "columnName": "name", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "description", + "columnName": "description", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "images.icon", + "columnName": "icon", + "affinity": "TEXT" + }, + { + "fieldPath": "images.list", + "columnName": "list", + "affinity": "TEXT" + }, + { + "fieldPath": "images.screenshot", + "columnName": "screenshot", + "affinity": "TEXT" + } + ], + "primaryKey": { + "autoGenerate": true, + "columnNames": [ + "platformEntryId" + ] + }, + "indices": [ + { + "name": "index_SyncedLockerEntryPlatform_lockerEntryId", + "unique": false, + "columnNames": [ + "lockerEntryId" + ], + "orders": [], + "createSql": "CREATE INDEX IF NOT EXISTS `index_SyncedLockerEntryPlatform_lockerEntryId` ON `${TABLE_NAME}` (`lockerEntryId`)" + } + ], + "foreignKeys": [ + { + "table": "SyncedLockerEntry", + "onDelete": "CASCADE", + "onUpdate": "NO ACTION", + "columns": [ + "lockerEntryId" + ], + "referencedColumns": [ + "id" + ] + } + ] + } + ], + "setupQueries": [ + "CREATE TABLE IF NOT EXISTS room_master_table (id INTEGER PRIMARY KEY,identity_hash TEXT)", + "INSERT OR REPLACE INTO room_master_table (id,identity_hash) VALUES(42, '0f71b8464e1fb6d1e3cd0f6754b90233')" + ] + } +} \ No newline at end of file diff --git a/android/shared/src/androidMain/kotlin/io/rebble/cobble/shared/di/koin.android.kt b/android/shared/src/androidMain/kotlin/io/rebble/cobble/shared/di/koin.android.kt index 47365e25..682dacd3 100644 --- a/android/shared/src/androidMain/kotlin/io/rebble/cobble/shared/di/koin.android.kt +++ b/android/shared/src/androidMain/kotlin/io/rebble/cobble/shared/di/koin.android.kt @@ -14,6 +14,7 @@ fun initKoin(context: Context) { dataStoreModule, androidModule, libpebbleModule, + voiceModule, dependenciesModule ) } diff --git a/android/shared/src/androidMain/kotlin/io/rebble/cobble/shared/domain/voice/NullDictationService.android.kt b/android/shared/src/androidMain/kotlin/io/rebble/cobble/shared/domain/voice/NullDictationService.android.kt new file mode 100644 index 00000000..0643d459 --- /dev/null +++ b/android/shared/src/androidMain/kotlin/io/rebble/cobble/shared/domain/voice/NullDictationService.android.kt @@ -0,0 +1,30 @@ +package io.rebble.cobble.shared.domain.voice + +import io.rebble.cobble.shared.Logging +import kotlinx.datetime.Clock +import org.koin.mp.KoinPlatformTools + +actual suspend fun writeRecording(encoderInfo: SpeexEncoderInfo, frames: List) { + val koin = KoinPlatformTools.defaultContext().get() + val context = koin.get() + val timestamp = Clock.System.now().epochSeconds + val file = context.getExternalFilesDir(null)!!.resolve("recording-$timestamp.spx") + file.outputStream().use { stream -> + frames.forEach { + stream.write(it.data) + } + } + Logging.d("Wrote recording to $file") + val metadataFile = context.getExternalFilesDir(null)!!.resolve("recording-$timestamp.json") + metadataFile.writeText( + """ + { + "version": "${encoderInfo.version}", + "sampleRate": ${encoderInfo.sampleRate}, + "bitRate": ${encoderInfo.bitRate}, + "bitstreamVersion": ${encoderInfo.bitstreamVersion}, + "frameSize": ${encoderInfo.frameSize} + } + """.trimIndent() + ) +} \ No newline at end of file diff --git a/android/shared/src/commonMain/kotlin/io/rebble/cobble/shared/di/VoiceModule.kt b/android/shared/src/commonMain/kotlin/io/rebble/cobble/shared/di/VoiceModule.kt new file mode 100644 index 00000000..f1c8e9d6 --- /dev/null +++ b/android/shared/src/commonMain/kotlin/io/rebble/cobble/shared/di/VoiceModule.kt @@ -0,0 +1,11 @@ +package io.rebble.cobble.shared.di + +import io.rebble.cobble.shared.domain.voice.DictationService +import io.rebble.cobble.shared.domain.voice.NullDictationService +import org.koin.core.module.dsl.factoryOf +import org.koin.dsl.bind +import org.koin.dsl.module + +val voiceModule = module { + factoryOf(::NullDictationService) bind DictationService::class +} \ No newline at end of file diff --git a/android/shared/src/commonMain/kotlin/io/rebble/cobble/shared/domain/voice/DictationService.kt b/android/shared/src/commonMain/kotlin/io/rebble/cobble/shared/domain/voice/DictationService.kt index d18bc36f..b427fe0d 100644 --- a/android/shared/src/commonMain/kotlin/io/rebble/cobble/shared/domain/voice/DictationService.kt +++ b/android/shared/src/commonMain/kotlin/io/rebble/cobble/shared/domain/voice/DictationService.kt @@ -3,5 +3,5 @@ package io.rebble.cobble.shared.domain.voice import kotlinx.coroutines.flow.Flow interface DictationService { - fun handleSpeechStream(audioStreamFrames: Flow): Flow + fun handleSpeechStream(speexEncoderInfo: SpeexEncoderInfo, audioStreamFrames: Flow): Flow } \ No newline at end of file diff --git a/android/shared/src/commonMain/kotlin/io/rebble/cobble/shared/domain/voice/DictationServiceResponse.kt b/android/shared/src/commonMain/kotlin/io/rebble/cobble/shared/domain/voice/DictationServiceResponse.kt index f9453e40..1822a66c 100644 --- a/android/shared/src/commonMain/kotlin/io/rebble/cobble/shared/domain/voice/DictationServiceResponse.kt +++ b/android/shared/src/commonMain/kotlin/io/rebble/cobble/shared/domain/voice/DictationServiceResponse.kt @@ -4,12 +4,16 @@ import io.rebble.libpebblecommon.packets.Result open class DictationServiceResponse { object Ready: DictationServiceResponse() - class Error(val result: Result): DictationServiceResponse() - class Transcription(val words: List): DictationServiceResponse() + data class Error(val result: Result): DictationServiceResponse() + data class Transcription(val sentences: List>): DictationServiceResponse() object Complete : DictationServiceResponse() } data class Word( val text: String, val confidence: UByte -) \ No newline at end of file +) { + init { + require(confidence in 0u..100u) { "Confidence must be between 0 and 100" } + } +} \ No newline at end of file diff --git a/android/shared/src/commonMain/kotlin/io/rebble/cobble/shared/domain/voice/NullDictationService.kt b/android/shared/src/commonMain/kotlin/io/rebble/cobble/shared/domain/voice/NullDictationService.kt new file mode 100644 index 00000000..1722bcc7 --- /dev/null +++ b/android/shared/src/commonMain/kotlin/io/rebble/cobble/shared/domain/voice/NullDictationService.kt @@ -0,0 +1,34 @@ +package io.rebble.cobble.shared.domain.voice + +import io.rebble.cobble.shared.Logging +import kotlinx.coroutines.* +import kotlinx.coroutines.flow.* +import kotlinx.datetime.Clock +import kotlin.time.Duration.Companion.seconds + +class NullDictationService: DictationService { + override fun handleSpeechStream(speexEncoderInfo: SpeexEncoderInfo, audioStreamFrames: Flow) = flow { + val frames = mutableListOf() + audioStreamFrames.onStart { + emit(DictationServiceResponse.Ready) + } + .onEach { + Logging.v("AudioStreamFrame: $it") + } + .collect { + if (it is AudioStreamFrame.Stop) { + emit(DictationServiceResponse.Transcription(listOf( + "Hello World!".split(" ").map { word -> Word(word, 100u) }) + )) + withContext(Dispatchers.IO) { + writeRecording(speexEncoderInfo, frames) + } + emit(DictationServiceResponse.Complete) + } else if (it is AudioStreamFrame.AudioData) { + frames.add(it) + } + } + } +} + +expect suspend fun writeRecording(encoderInfo: SpeexEncoderInfo, frames: List) \ No newline at end of file diff --git a/android/shared/src/commonMain/kotlin/io/rebble/cobble/shared/handlers/AudioStreamHandler.kt b/android/shared/src/commonMain/kotlin/io/rebble/cobble/shared/handlers/AudioStreamHandler.kt index 5917606a..e42125ab 100644 --- a/android/shared/src/commonMain/kotlin/io/rebble/cobble/shared/handlers/AudioStreamHandler.kt +++ b/android/shared/src/commonMain/kotlin/io/rebble/cobble/shared/handlers/AudioStreamHandler.kt @@ -32,7 +32,7 @@ class AudioStreamHandler( return } if (pebbleDevice.activeVoiceSession.value?.sessionId != message.sessionId.get().toInt()) { - Logging.e("Received audio stream data transfer for different session ID") + Logging.e("Received audio stream data transfer for different session ID (expected ${pebbleDevice.activeVoiceSession.value?.sessionId}, got ${message.sessionId.get()})") pebbleDevice.audioStreamService.send(AudioStream.StopTransfer(message.sessionId.get())) return } diff --git a/android/shared/src/commonMain/kotlin/io/rebble/cobble/shared/handlers/VoiceSessionHandler.kt b/android/shared/src/commonMain/kotlin/io/rebble/cobble/shared/handlers/VoiceSessionHandler.kt index e7a48b3b..32345f19 100644 --- a/android/shared/src/commonMain/kotlin/io/rebble/cobble/shared/handlers/VoiceSessionHandler.kt +++ b/android/shared/src/commonMain/kotlin/io/rebble/cobble/shared/handlers/VoiceSessionHandler.kt @@ -8,10 +8,7 @@ import io.rebble.cobble.shared.domain.voice.SpeexEncoderInfo import io.rebble.cobble.shared.domain.voice.VoiceSession import io.rebble.libpebblecommon.packets.* import io.rebble.libpebblecommon.util.DataBuffer -import kotlinx.coroutines.flow.filterNotNull -import kotlinx.coroutines.flow.first -import kotlinx.coroutines.flow.onStart -import kotlinx.coroutines.flow.takeWhile +import kotlinx.coroutines.flow.* import kotlinx.coroutines.launch import org.koin.core.component.KoinComponent import org.koin.core.component.inject @@ -28,6 +25,25 @@ class VoiceSessionHandler( } } + private fun makeTranscription(sentences: List>): VoiceAttribute { + val data = VoiceAttribute.Transcription( + sentences = sentences.map { sentence -> + Sentence( + words = sentence.map { word -> + Word( + word.confidence, + word.text + ) + } + ) + } + ) + return VoiceAttribute( + id = VoiceAttributeType.Transcription.value, + content = data + ) + } + private suspend fun listenForVoiceSessions() { for (message in pebbleDevice.voiceService.receivedMessages) { when (message) { @@ -56,11 +72,15 @@ class VoiceSessionHandler( Logging.d("Received voice session: $voiceSession") var sentReady = false - dictationService.handleSpeechStream(voiceSession.audioStreamFrames) + dictationService.handleSpeechStream(voiceSession.encoderInfo, voiceSession.audioStreamFrames) .takeWhile { it !is DictationServiceResponse.Complete } + .onEach { + Logging.v("DictationServiceResponse: $it") + } .collect { when (it) { is DictationServiceResponse.Ready -> { + pebbleDevice.activeVoiceSession.value = voiceSession pebbleDevice.voiceService.send(SessionSetupResult( sessionType = SessionType.Dictation, result = Result.Success @@ -82,26 +102,17 @@ class VoiceSessionHandler( } } is DictationServiceResponse.Transcription -> { - pebbleDevice.voiceService.send(DictationResult( + val a = DictationResult( voiceSession.sessionId.toUShort(), Result.Success, - buildList { - val attr = VoiceAttribute() - val data = VoiceAttribute.Transcription( - words = it.words.map { word -> - Word( - word.confidence, - word.text - ) - } - ) - } - )) + listOf( + makeTranscription(it.sentences) + ) + ) + pebbleDevice.voiceService.send(a) } } } - - pebbleDevice.activeVoiceSession.value = voiceSession } } diff --git a/android/shared/src/iosMain/kotlin/io/rebble/cobble/shared/domain/voice/NullDictationService.ios.kt b/android/shared/src/iosMain/kotlin/io/rebble/cobble/shared/domain/voice/NullDictationService.ios.kt new file mode 100644 index 00000000..2f232177 --- /dev/null +++ b/android/shared/src/iosMain/kotlin/io/rebble/cobble/shared/domain/voice/NullDictationService.ios.kt @@ -0,0 +1,5 @@ +package io.rebble.cobble.shared.domain.voice + +actual suspend fun writeRecording(encoderInfo: SpeexEncoderInfo, frames: List) { + TODO() +} \ No newline at end of file