Add Android WearOS demo (#319)

* Initial commit after creating the project

* First working version

* add a medium model

android/.gitignore

@@ -0,0 +1,2 @@
+!*.jar
+*.so

android/SherpaNcnn/app/src/main/java/com/k2fsa/sherpa/ncnn/SherpaNcnn.kt

@@ -126,6 +126,11 @@ fun getDecoderConfig(method: String, numActivePaths: Int): DecoderConfig {
 4 - https://huggingface.co/shaojieli/sherpa-ncnn-streaming-zipformer-fr-2023-04-14
     This model supports only French
 
+5 - https://github.com/k2-fsa/sherpa-ncnn/releases/download/models/sherpa-ncnn-streaming-zipformer-zh-14M-2023-02-23.tar.bz2
+    This is a small model and supports only Chinese
+6 - https://github.com/k2-fsa/sherpa-ncnn/releases/download/models/sherpa-ncnn-streaming-zipformer-small-bilingual-zh-en-2023-02-16.tar.bz2
+    This is a medium model and supports only Chinese
+
 Please follow
 https://k2-fsa.github.io/sherpa/ncnn/pretrained_models/index.html
 to add more pre-trained models
@@ -206,6 +211,36 @@ fun getModelConfig(type: Int, useGPU: Boolean): ModelConfig? {
                 useGPU = useGPU,
             )
         }
+
+        5 -> {
+            val modelDir = "sherpa-ncnn-streaming-zipformer-zh-14M-2023-02-23"
+            return ModelConfig(
+                encoderParam = "$modelDir/encoder_jit_trace-pnnx.ncnn.param",
+                encoderBin = "$modelDir/encoder_jit_trace-pnnx.ncnn.bin",
+                decoderParam = "$modelDir/decoder_jit_trace-pnnx.ncnn.param",
+                decoderBin = "$modelDir/decoder_jit_trace-pnnx.ncnn.bin",
+                joinerParam = "$modelDir/joiner_jit_trace-pnnx.ncnn.param",
+                joinerBin = "$modelDir/joiner_jit_trace-pnnx.ncnn.bin",
+                tokens = "$modelDir/tokens.txt",
+                numThreads = 2,
+                useGPU = useGPU,
+            )
+        }
+
+        6 -> {
+            val modelDir = "sherpa-ncnn-streaming-zipformer-small-bilingual-zh-en-2023-02-16/96"
+            return ModelConfig(
+                encoderParam = "$modelDir/encoder_jit_trace-pnnx.ncnn.param",
+                encoderBin = "$modelDir/encoder_jit_trace-pnnx.ncnn.bin",
+                decoderParam = "$modelDir/decoder_jit_trace-pnnx.ncnn.param",
+                decoderBin = "$modelDir/decoder_jit_trace-pnnx.ncnn.bin",
+                joinerParam = "$modelDir/joiner_jit_trace-pnnx.ncnn.param",
+                joinerBin = "$modelDir/joiner_jit_trace-pnnx.ncnn.bin",
+                tokens = "$modelDir/tokens.txt",
+                numThreads = 2,
+                useGPU = useGPU,
+            )
+        }
     }
     return null
 }

android/SherpaNcnnWearOS/.gitignore

@@ -0,0 +1,15 @@
+*.iml
+.gradle
+/local.properties
+/.idea/caches
+/.idea/libraries
+/.idea/modules.xml
+/.idea/workspace.xml
+/.idea/navEditor.xml
+/.idea/assetWizardSettings.xml
+.DS_Store
+/build
+/captures
+.externalNativeBuild
+.cxx
+local.properties

android/SherpaNcnnWearOS/app/.gitignore

@@ -0,0 +1 @@
+/build

android/SherpaNcnnWearOS/app/build.gradle.kts

@@ -0,0 +1,65 @@
+plugins {
+    id("com.android.application")
+    id("org.jetbrains.kotlin.android")
+}
+
+android {
+    namespace = "com.k2fsa.sherpa.ncnn.wear.os"
+    compileSdk = 34
+
+    defaultConfig {
+        applicationId = "com.k2fsa.sherpa.ncnn.wear.os"
+        minSdk = 26
+        targetSdk = 34
+        versionCode = 1
+        versionName = "1.0"
+        vectorDrawables {
+            useSupportLibrary = true
+        }
+
+    }
+
+    buildTypes {
+        release {
+            isMinifyEnabled = false
+            proguardFiles(
+                getDefaultProguardFile("proguard-android-optimize.txt"),
+                "proguard-rules.pro"
+            )
+        }
+    }
+    compileOptions {
+        sourceCompatibility = JavaVersion.VERSION_1_8
+        targetCompatibility = JavaVersion.VERSION_1_8
+    }
+    kotlinOptions {
+        jvmTarget = "1.8"
+    }
+    buildFeatures {
+        compose = true
+    }
+    composeOptions {
+        kotlinCompilerExtensionVersion = "1.5.1"
+    }
+    packaging {
+        resources {
+            excludes += "/META-INF/{AL2.0,LGPL2.1}"
+        }
+    }
+}
+
+dependencies {
+
+    implementation("com.google.android.gms:play-services-wearable:18.1.0")
+    implementation(platform("androidx.compose:compose-bom:2023.08.00"))
+    implementation("androidx.compose.ui:ui")
+    implementation("androidx.compose.ui:ui-tooling-preview")
+    implementation("androidx.wear.compose:compose-material:1.1.2")
+    implementation("androidx.wear.compose:compose-foundation:1.1.2")
+    implementation("androidx.activity:activity-compose:1.7.2")
+    implementation("androidx.core:core-splashscreen:1.0.1")
+    androidTestImplementation(platform("androidx.compose:compose-bom:2023.08.00"))
+    androidTestImplementation("androidx.compose.ui:ui-test-junit4")
+    debugImplementation("androidx.compose.ui:ui-tooling")
+    debugImplementation("androidx.compose.ui:ui-test-manifest")
+}

android/SherpaNcnnWearOS/app/lint.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<lint>
+    <!-- Ignore the IconLocation for the Tile preview images -->
+    <issue id="IconLocation">
+        <ignore path="res/drawable/tile_preview.png" />
+        <ignore path="res/drawable-round/tile_preview.png" />
+    </issue>
+</lint>

android/SherpaNcnnWearOS/app/proguard-rules.pro

@@ -0,0 +1,21 @@
+# Add project specific ProGuard rules here.
+# You can control the set of applied configuration files using the
+# proguardFiles setting in build.gradle.
+#
+# For more details, see
+#   http://developer.android.com/guide/developing/tools/proguard.html
+
+# If your project uses WebView with JS, uncomment the following
+# and specify the fully qualified class name to the JavaScript interface
+# class:
+#-keepclassmembers class fqcn.of.javascript.interface.for.webview {
+#   public *;
+#}
+
+# Uncomment this to preserve the line number information for
+# debugging stack traces.
+#-keepattributes SourceFile,LineNumberTable
+
+# If you keep the line number information, uncomment this to
+# hide the original source file name.
+#-renamesourcefileattribute SourceFile

android/SherpaNcnnWearOS/app/src/main/AndroidManifest.xml

@@ -0,0 +1,40 @@
+<?xml version="1.0" encoding="utf-8"?>
+<manifest xmlns:android="http://schemas.android.com/apk/res/android">
+
+    <uses-permission android:name="android.permission.WAKE_LOCK" />
+    <uses-permission android:name="android.permission.RECORD_AUDIO" />
+
+    <uses-feature android:name="android.hardware.type.watch" />
+
+    <application
+        android:allowBackup="true"
+        android:icon="@mipmap/ic_launcher"
+        android:label="@string/app_name"
+        android:supportsRtl="true"
+        android:theme="@android:style/Theme.DeviceDefault">
+        <uses-library
+            android:name="com.google.android.wearable"
+            android:required="true" />
+
+        <!--
+               Set to true if your app is Standalone, that is, it does not require the handheld
+               app to run.
+        -->
+        <meta-data
+            android:name="com.google.android.wearable.standalone"
+            android:value="true" />
+
+        <activity
+            android:name=".presentation.MainActivity"
+            android:exported="true"
+            android:taskAffinity=""
+            android:theme="@style/MainActivityTheme.Starting">
+            <intent-filter>
+                <action android:name="android.intent.action.MAIN" />
+
+                <category android:name="android.intent.category.LAUNCHER" />
+            </intent-filter>
+        </activity>
+    </application>
+
+</manifest>

android/SherpaNcnnWearOS/app/src/main/java/com/k2fsa/sherpa/ncnn/wear/os/presentation/HomeScreen.kt

@@ -0,0 +1,172 @@
+package com.k2fsa.sherpa.ncnn.wear.os.presentation
+
+import android.Manifest
+import android.app.Activity
+import android.content.pm.PackageManager
+import android.media.AudioFormat
+import android.media.AudioRecord
+import android.media.MediaRecorder
+import android.util.Log
+import androidx.compose.foundation.background
+import androidx.compose.foundation.layout.Arrangement
+import androidx.compose.foundation.layout.Box
+import androidx.compose.foundation.layout.Column
+import androidx.compose.foundation.layout.Spacer
+import androidx.compose.foundation.layout.fillMaxSize
+import androidx.compose.foundation.layout.fillMaxWidth
+import androidx.compose.foundation.layout.height
+import androidx.compose.runtime.Composable
+import androidx.compose.runtime.getValue
+import androidx.compose.runtime.mutableStateOf
+import androidx.compose.runtime.remember
+import androidx.compose.runtime.setValue
+import androidx.compose.ui.Alignment
+import androidx.compose.ui.Modifier
+import androidx.compose.ui.platform.LocalContext
+import androidx.compose.ui.text.style.TextAlign
+import androidx.compose.ui.unit.dp
+import androidx.compose.ui.unit.sp
+import androidx.core.app.ActivityCompat
+import androidx.wear.compose.material.Button
+import androidx.wear.compose.material.MaterialTheme
+import androidx.wear.compose.material.Text
+import com.k2fsa.sherpa.ncnn.wear.os.presentation.theme.SherpaNcnnWearOSTheme
+import kotlin.concurrent.thread
+
+private var audioRecord: AudioRecord? = null
+private val sampleRateInHz = 16000
+
+@Composable
+fun HomeScreen() {
+    val activity = LocalContext.current as Activity
+    var firstTime by remember { mutableStateOf(true) }
+    var isStarted by remember { mutableStateOf(false) }
+    var result by remember { mutableStateOf("") }
+    val onButtonClick: () -> Unit = {
+        firstTime = false;
+        isStarted = !isStarted
+        if (isStarted) {
+            if (ActivityCompat.checkSelfPermission(
+                    activity,
+                    Manifest.permission.RECORD_AUDIO
+                ) != PackageManager.PERMISSION_GRANTED
+            ) {
+                Log.i(TAG, "Recording is not allowed")
+            } else {
+                // recording is allowed
+                val audioSource = MediaRecorder.AudioSource.MIC
+                val channelConfig = AudioFormat.CHANNEL_IN_MONO
+                val audioFormat = AudioFormat.ENCODING_PCM_16BIT
+                val numBytes =
+                    AudioRecord.getMinBufferSize(sampleRateInHz, channelConfig, audioFormat)
+
+                audioRecord = AudioRecord(
+                    audioSource,
+                    sampleRateInHz,
+                    AudioFormat.CHANNEL_IN_MONO,
+                    AudioFormat.ENCODING_PCM_16BIT,
+                    numBytes * 2 // a sample has two bytes as we are using 16-bit PCM
+                )
+
+                thread(true) {
+                    Log.i(TAG, "processing samples")
+                    val interval = 0.1 // i.e., 100 ms
+                    val bufferSize = (interval * sampleRateInHz).toInt() // in samples
+                    val buffer = ShortArray(bufferSize)
+                    audioRecord?.let {
+                        it.startRecording()
+                        while (isStarted) {
+                            val ret = audioRecord?.read(buffer, 0, buffer.size)
+                            ret?.let { n ->
+                                val samples = FloatArray(n) { buffer[it] / 32768.0f }
+                                Recognizer.recognizer.acceptSamples(samples)
+                                while (Recognizer.recognizer.isReady()) {
+                                    Recognizer.recognizer.decode()
+                                }
+                                val isEndpoint = Recognizer.recognizer.isEndpoint()
+                                val text = Recognizer.recognizer.text
+                                if (isEndpoint) {
+                                    Recognizer.recognizer.reset()
+                                }
+                                Log.i(TAG, "text: $text")
+                                result = text
+                            }
+                        }
+                    }
+                    Log.i(TAG, "Stop recording")
+                }
+            }
+        }
+    }
+
+    SherpaNcnnWearOSTheme {
+        Box(
+            modifier = Modifier
+                .fillMaxSize()
+                .background(MaterialTheme.colors.background),
+            contentAlignment = Alignment.Center
+        ) {
+            Column(
+                // verticalArrangement = Arrangement.Center,
+                horizontalAlignment = Alignment.CenterHorizontally
+            ) {
+                Spacer(modifier = Modifier.height(16.dp))
+
+                if (firstTime) {
+                    ShowMessage()
+                } else {
+                    ShowResult(result)
+                }
+                Spacer(modifier = Modifier.height(32.dp))
+                Button(
+                    onClick = onButtonClick
+                ) {
+
+                    if (isStarted) {
+                        Text("Stop")
+                    } else {
+                        Text("Start")
+                    }
+                }
+                if (!firstTime) {
+                    Spacer(modifier = Modifier.height(16.dp))
+                    Text(
+                        "Next-gen Kaldi + ncnn\nASR",
+                        modifier = Modifier.fillMaxWidth(),
+                        textAlign = TextAlign.Center,
+                        color = MaterialTheme.colors.primary,
+                        fontSize = 10.sp
+                    )
+                }
+            }
+        }
+    }
+}
+
+@Composable
+fun ShowMessage() {
+    val msg = "Real-time\nspeech recognition\nwith\nNext-gen Kaldi + ncnn"
+    Text(
+        modifier = Modifier.fillMaxWidth(),
+        textAlign = TextAlign.Center,
+        color = MaterialTheme.colors.primary,
+        text = msg,
+    )
+}
+
+@Composable
+fun ShowResult(result: String) {
+    var msg: String = result
+    if (msg.length > 10) {
+        val n = 5;
+        val first = result.take(n);
+        val last = result.takeLast(result.length - n)
+        msg = "${first}\n${last}"
+    }
+    Text(
+        modifier = Modifier.fillMaxWidth(),
+        textAlign = TextAlign.Center,
+        color = MaterialTheme.colors.primary,
+        text = msg,
+    )
+}