Merge pull request #57 from solaoi/main

update to v0.6.0

README.md

@@ -4,11 +4,13 @@
 [![GitHub release (latest by date)](https://img.shields.io/github/v/release/solaoi/lycoris)](https://github.com/solaoi/lycoris/releases)
 [![GitHub Sponsors](https://img.shields.io/github/sponsors/solaoi?color=db61a2)](https://github.com/sponsors/solaoi)
 
-外部データ通信無しで、リアルタイム音声認識で文字起こしを行う、デスクトップアプリケーションを作りました。
+外部データ通信無しで、リアルタイム音声認識で文字起こしを行う音声ノートアプリケーションです。
 
-文字起こしと同じタイムライン上で、メモを残すこともできます。
-
-<img width="802" alt="スクリーンショット 2022-09-17 18 31 40" src="https://user-images.githubusercontent.com/46414076/190865611-0e107efd-3112-4229-bab7-ea7e904718ee.png">
+<div align="center">
+  <a href="https://github.com/solaoi/lycoris">
+    <img width="600" alt="lycoris" src="https://user-images.githubusercontent.com/46414076/227781834-2eeaea6f-fae6-4607-8862-4ca74a4416b9.png">
+  </a>
+</div>
 
 ## 対応環境
 

package.json

@@ -22,7 +22,9 @@
     "react-dom": "^18.2.0",
     "react-h5-audio-player": "^3.8.6",
     "recoil": "^0.7.7",
-    "tauri-plugin-sql-api": "github:tauri-apps/tauri-plugin-sql#release"
+    "tauri-plugin-sql-api": "github:tauri-apps/tauri-plugin-sql#release",
+    "zenn-content-css": "^0.1.141",
+    "zenn-markdown-html": "^0.1.141"
   },
   "devDependencies": {
     "@tauri-apps/cli": "^1.2.3",

src-tauri/src/main.rs

@@ -6,6 +6,7 @@
 use crossbeam_channel::Sender;
 use module::model_type_vosk::ModelTypeVosk;
 use module::model_type_whisper::ModelTypeWhisper;
+use module::transcription::TraceCompletion;
 use tauri::http::HttpRange;
 use tauri::http::ResponseBuilder;
 use tauri::Manager;
@@ -93,6 +94,45 @@ fn stop_command(state: State<'_, RecordState>) {
     }
 }
 
+#[tauri::command]
+fn start_trace_command(
+    state: State<'_, RecordState>,
+    window: tauri::Window,
+    speaker_language: String,
+    transcription_accuracy: String,
+    note_id: u64,
+) {
+    let mut lock = state.0.lock().unwrap();
+    let (stop_convert_tx, stop_convert_rx) = unbounded();
+    *lock = Some(stop_convert_tx);
+
+    std::thread::spawn(move || {
+        let mut transcription = module::transcription::Transcription::new(
+            window.app_handle(),
+            transcription_accuracy,
+            speaker_language,
+            note_id,
+        );
+        transcription.start(stop_convert_rx, true);
+    });
+}
+
+#[tauri::command]
+fn stop_trace_command(state: State<'_, RecordState>, window: tauri::Window) {
+    let mut lock = state.0.lock().unwrap();
+    if let Some(stop_convert_tx) = lock.take() {
+        stop_convert_tx.send(()).unwrap_or_else(|_| {
+            window
+                .app_handle()
+                .emit_all(
+                    "traceCompletion",
+                    TraceCompletion {},
+                )
+                .unwrap();
+        })
+    }
+}
+
 fn main() {
     tauri::Builder::default()
         .register_uri_scheme_protocol("stream", move |_app, request| {
@@ -158,6 +198,8 @@ fn main() {
             list_devices_command,
             start_command,
             stop_command,
+            start_trace_command,
+            stop_trace_command,
         ])
         .run(tauri::generate_context!())
         .expect("error while running tauri application");

src-tauri/src/module/mod.rs

@@ -7,5 +7,5 @@ mod recognizer;
 pub mod record;
 mod sqlite;
 mod transcriber;
-mod transcription;
+pub mod transcription;
 mod writer;

src-tauri/src/module/record.rs

@@ -139,6 +139,7 @@ impl Record {
         let (stop_writer_tx, stop_writer_rx) = sync_channel(1);
         let is_converting = Arc::new(Mutex::new(false));
         let (stop_convert_tx, stop_convert_rx) = unbounded();
+        let is_no_transcription = transcription_accuracy == "off";
 
         let app_handle = self.app_handle.clone();
         thread::spawn(move || loop {
@@ -177,7 +178,7 @@ impl Record {
                         .unwrap()
                         .replace(Writer::build(&audio_path.to_str().expect("error"), spec));
 
-                    if !*is_converting.lock().unwrap() {
+                    if !is_no_transcription && !*is_converting.lock().unwrap() {
                         let is_converting_clone = Arc::clone(&is_converting);
                         let app_handle_clone = app_handle.clone();
                         let stop_convert_rx_clone = stop_convert_rx.clone();
@@ -191,9 +192,9 @@ impl Record {
                                 app_handle_clone,
                                 transcription_accuracy_clone,
                                 speaker_language_clone,
-                                note_id
+                                note_id,
                             );
-                            transcription.start(stop_convert_rx_clone);
+                            transcription.start(stop_convert_rx_clone, false);
                             let mut lock = is_converting_clone.lock().unwrap();
                             *lock = false;
                             drop(lock);
@@ -214,6 +215,10 @@ impl Record {
             .expect("failed to receive the message");
         drop(stream);
         stop_writer_tx.send(()).unwrap();
-        stop_convert_tx.send(()).unwrap();
+        if !is_no_transcription {
+            stop_convert_tx.send(()).unwrap();
+        } else {
+            drop(stop_convert_tx)
+        }
     }
 }

src-tauri/src/module/transcription.rs

@@ -7,6 +7,9 @@ use samplerate_rs::{convert, ConverterType};
 use tauri::{AppHandle, Manager};
 use whisper_rs::WhisperContext;
 
+#[derive(Debug, Clone, serde::Serialize)]
+pub struct TraceCompletion {}
+
 pub struct Transcription {
     app_handle: AppHandle,
     sqlite: Sqlite,
@@ -32,9 +35,40 @@ impl Transcription {
         }
     }
 
-    pub fn start(&mut self, stop_convert_rx: Receiver<()>) {
+    pub fn start(&mut self, stop_convert_rx: Receiver<()>, is_continuous: bool) {
         while Self::convert(self).is_ok() {
+            if is_continuous {
+                let vosk_speech = self.sqlite.select_vosk(self.note_id);
+                if vosk_speech.is_err() {
+                    self.app_handle
+                        .clone()
+                        .emit_all(
+                            "traceCompletion",
+                            TraceCompletion {},
+                        )
+                        .unwrap();
+                    break;
+                }
+            }
             if stop_convert_rx.try_recv().is_ok() {
+                let vosk_speech = self.sqlite.select_vosk(self.note_id);
+                if vosk_speech.is_err() {
+                    self.app_handle
+                        .clone()
+                        .emit_all(
+                            "traceCompletion",
+                            TraceCompletion {},
+                        )
+                        .unwrap();
+                } else {
+                    self.app_handle
+                        .clone()
+                        .emit_all(
+                            "traceUnCompletion",
+                            TraceCompletion {},
+                        )
+                        .unwrap();
+                }
                 break;
             }
         }
@@ -103,17 +137,7 @@ impl Transcription {
                         .ctx
                         .full_get_segment_text(i)
                         .expect("failed to get segment");
-                    let last = converted.last().unwrap().as_str();
-                    if segment != last
-                        && segment != "(音楽)"
-                        && segment != "[音楽]"
-                        && segment != "(音声)"
-                        && segment != "(小声)"
-                        && segment != "(EN)"
-                        && segment != "(笑)"
-                    {
-                        converted.push(segment.to_string());
-                    }
+                    converted.push(segment.to_string());
                 }
 
                 let updated = self

src-tauri/tauri.conf.json

@@ -8,7 +8,7 @@
   },
   "package": {
     "productName": "Lycoris",
-    "version": "0.5.0"
+    "version": "0.6.0"
   },
   "tauri": {
     "allowlist": {

src/App.tsx

@@ -8,7 +8,7 @@ function App() {
   return (
     <RecoilRoot>
       <Header />
-      <div style={{display: "flex"}}>
+      <div style={{ display: "flex" }}>
         <SideMenu />
         <Main />
       </div>

src/components/SideMenu.tsx

@@ -1,19 +1,33 @@
 import { invoke } from "@tauri-apps/api"
 import { listen } from "@tauri-apps/api/event"
-import { useEffect, useState } from "react"
+import { useEffect, useRef, useState } from "react"
 import { useRecoilState } from "recoil"
 import { notesState } from "../store/atoms/notesState"
 import { selectedNoteState } from "../store/atoms/selectedNoteState"
 import { DeleteCompletionType } from "../type/DeleteCompletion.type"
 import { SideMenuColumn } from "./organisms/SideMenuColumn"
 
+const KatakanaToHiragana = (katakanaValue: string | null | undefined): string => {
+    if (!katakanaValue) {
+        return '';
+    }
+
+    return katakanaValue.replace(/[\u30a1-\u30f6]/g, (substring: string): string => {
+        const hiraganaCharCode: number = substring.charCodeAt(0) - 0x60;
+        return String.fromCharCode(hiraganaCharCode);
+    });
+}
+
 const SideMenu = (): JSX.Element => {
+    const targetRef = useRef<HTMLDivElement>(null);
+    const [searchWord, setSearchWord] = useState("");
     const [notes, setNotes] = useRecoilState(notesState);
     const [isAdded, setAdded] = useState(false);
     const [selectedNote, setSelectedNote] = useRecoilState(selectedNoteState);
     useEffect(() => {
         if (!notes[notes.length - 1]?.id) {
             setAdded(true)
+            targetRef.current?.scrollIntoView({ behavior: 'smooth' })
         }
     }, [notes]);
     useEffect(() => {
@@ -38,7 +52,7 @@ const SideMenu = (): JSX.Element => {
         <div className="flex justify-between border-r" style={{ width: "320px", flexFlow: "column", height: `calc(100vh - 64px)` }}>
             <div className="flex" style={{ flexFlow: "column" }}>
                 <div className="bg-white flex items-center justify-center select-none text-xl" style={{ height: "64px" }}>
-                    <input type="text" placeholder="ノートを検索..." className="input input-bordered focus:outline-none" />
+                    <input type="text" placeholder="タイトル検索..." className="input input-bordered focus:outline-none" value={searchWord} onChange={e => setSearchWord(e.target.value)} />
                     <button type="button" className="bg-base-200 hover:bg-base-300 focus:outline-none rounded-lg px-5 m-1"
                         style={{ height: "48px" }}
                         onClick={() => {
@@ -62,7 +76,13 @@ const SideMenu = (): JSX.Element => {
                     </button>
                 </div>
                 <div className="overflow-auto" style={{ height: `calc(100vh - 128px)` }} >
-                    {notes.map((note, i) => {
+                    <div ref={targetRef} />
+                    {notes.filter(note => {
+                        if (searchWord === "") {
+                            return true;
+                        }
+                        return KatakanaToHiragana(note.note_title.toLowerCase()).includes(KatakanaToHiragana(searchWord.toLowerCase()))
+                    }).map((note, i) => {
                         return (
                             <SideMenuColumn key={`note_${note.id}`} note={note} deleteAction={deleteAction(note.id!)} isAdded={i === 0 ? isAdded : false} setAdded={setAdded} />
                         )

src/components/molecules/MemoFilterButton.tsx

@@ -9,7 +9,10 @@ const MemoFilterButton = (): JSX.Element => {
 
     return (
         <button className={`select-none badge badge-md gap-2 text-white border-none ${filterd ? "bg-orange-900" : "bg-orange-400"}`} onClick={click}>
-            メモのみ表示
+            <svg width="8" height="8" viewBox="0, 0, 8, 8" xmlns="http://www.w3.org/2000/svg">
+                <rect width="8" height="8" style={{ "fill": "white" }} />
+            </svg>
+            メモ
         </button>
     )
 }

src/components/molecules/RecordStartButton.tsx

@@ -6,6 +6,7 @@ import { speakerLanguageState } from '../../store/atoms/speakerLanguageState';
 import { transcriptionAccuracyState } from '../../store/atoms/transcriptionAccuracyState';
 import { selectedNoteState } from '../../store/atoms/selectedNoteState';
 import { recordingNoteState } from '../../store/atoms/recordingNoteState';
+import { tracingState } from '../../store/atoms/tracingState';
 
 const RecordStartButton = (): JSX.Element => {
     const deviceLabel = useRecoilValue(audioDeviceState)
@@ -14,14 +15,15 @@ const RecordStartButton = (): JSX.Element => {
     const [isRecording, setRecording] = useRecoilState(recordState)
     const selectedNote = useRecoilValue(selectedNoteState)
     const [recordingNote, setRecordingNote] = useRecoilState(recordingNoteState)
+    const isTracing = useRecoilValue(tracingState);
     const click = () => {
         setRecording(true)
         setRecordingNote(selectedNote!.note_id)
         invoke('start_command', { deviceLabel, speakerLanguage, transcriptionAccuracy, noteId: selectedNote!.note_id })
     }
 
     return (
-        <button className="btn gap-2 glass text-primary" disabled={deviceLabel === null || speakerLanguage === null || (isRecording && recordingNote !== selectedNote?.note_id)} onClick={click}>
+        <button className="btn gap-2 glass text-primary" disabled={deviceLabel === null || speakerLanguage === null || isTracing || (isRecording && recordingNote !== selectedNote?.note_id)} onClick={click}>
             <svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" strokeWidth={1.5} stroke="currentColor" className="w-6 h-6">
                 <path strokeLinecap="round" strokeLinejoin="round" d="M12 18.75a6 6 0 006-6v-1.5m-6 7.5a6 6 0 01-6-6v-1.5m6 7.5v3.75m-3.75 0h7.5M12 15.75a3 3 0 01-3-3V4.5a3 3 0 116 0v8.25a3 3 0 01-3 3z" />
             </svg>

src/components/molecules/Speech.tsx

@@ -14,19 +14,15 @@ type SpeechProps = {
 const Speech = (props: SpeechProps): JSX.Element => {
     const { model, date, content, wav } = props
     const [isHover, setHover] = useState(false)
-    const [lazyHoverId, setLazyHoverId] = useState(0)
-    const [isLazyHover, setLazyHover] = useState(false)
     const targetRef = useRef(null);
     const { getElementProperty } = useGetElementProperty<HTMLDivElement>(targetRef);
     const width = getElementProperty("width");
     const bottom = getElementProperty("bottom");
     const leave = () => {
         setHover(false);
-        clearTimeout(lazyHoverId);
-        setLazyHover(false);
     }
     const scroll = () => {
-        setLazyHover(false);
+        setHover(false);
     }
     useEffect(() => {
         window.addEventListener('scroll', scroll)
@@ -35,14 +31,9 @@ const Speech = (props: SpeechProps): JSX.Element => {
 
     return (
         <div onMouseLeave={leave} >
-            <div className={"flex mb-1 cursor-pointer" + (isHover && " bg-gray-400 text-white rounded")}
-                onClick={() => setLazyHover(true)}
-                onMouseEnter={() => {
-                    setHover(true);
-                    if (window.innerHeight - bottom > 100) {
-                        setLazyHoverId(setTimeout(() => { setLazyHover(true) }, 1000));
-                    }
-                }} ref={targetRef} >
+            <div className={"flex mb-1 cursor-pointer hover:bg-gray-400 hover:text-white hover:rounded"}
+                onClick={() => setHover(true)}
+                ref={targetRef} >
                 <div className="w-16 pl-2 flex-none">{date}</div>
                 <div style={{ paddingTop: "0.5rem", paddingRight: "10px" }}>
                     <svg width="8" height="8" viewBox="0, 0, 8, 8" xmlns="http://www.w3.org/2000/svg">
@@ -51,7 +42,7 @@ const Speech = (props: SpeechProps): JSX.Element => {
                 </div>
                 <div className="pr-2">{content}</div>
             </div>
-            {isLazyHover &&
+            {isHover &&
                 <AudioPlayer filePath={convertFileSrc(wav, "stream")}
                     className="animate-fade-in"
                     style={{