Add extension for uLipSync

Assets/Mochineko/FacialExpressions.Extensions/VOICEVOX/package.json

@@ -1,11 +1,11 @@
 {
   "name": "com.mochineko.facial-expressions.extensions.voicevox",
-  "version": "0.3.2",
+  "version": "0.3.3",
   "displayName": "Facial Expressions / VOICEVOX",
   "description": "Facial expressions extension for VOICEVOX.",
   "unity": "2021.3",
   "author": {
     "name": "Mochineko",
     "email": "t.o.e.4315@gmail.com"
-  }    
+  }
 }

Assets/Mochineko/FacialExpressions.Extensions/VRM/package.json

@@ -1,6 +1,6 @@
 {
   "name": "com.mochineko.facial-expressions.extensions.vrm",
-  "version": "0.3.2",
+  "version": "0.3.3",
   "displayName": "Facial Expressions / VRM",
   "description": "Facial expressions extension for VRM.",
   "unity": "2021.3",

Assets/Mochineko/FacialExpressions.Extensions/uLipSync/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 mochineko
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

Assets/Mochineko/FacialExpressions.Extensions/uLipSync/Mochineko.FacialExpressions.Extensions.uLipSync.asmdef

@@ -0,0 +1,18 @@
+{
+    "name": "Mochineko.FacialExpressions.Extensions.uLipSync",
+    "rootNamespace": "",
+    "references": [
+        "GUID:0e0dbf6ed860b4d57b738a3d1faef12d",
+        "GUID:d444a9c79d21fec4181e6e12dc2e9706",
+        "GUID:560b04d1a97f54a4e82edc0cbbb69285"
+    ],
+    "includePlatforms": [],
+    "excludePlatforms": [],
+    "allowUnsafeCode": false,
+    "overrideReferences": true,
+    "precompiledReferences": [],
+    "autoReferenced": false,
+    "defineConstraints": [],
+    "versionDefines": [],
+    "noEngineReferences": false
+}

Assets/Mochineko/FacialExpressions.Extensions/uLipSync/ULipSyncAnimator.cs

@@ -0,0 +1,94 @@
+#nullable enable
+using System;
+using System.Collections.Generic;
+using Mochineko.FacialExpressions.LipSync;
+using UniRx;
+using UnityEngine;
+
+namespace Mochineko.FacialExpressions.Extensions.uLipSync
+{
+    public sealed class ULipSyncAnimator : IDisposable
+    {
+        private readonly FollowingLipAnimator followingLipAnimator;
+        private readonly global::uLipSync.uLipSync uLipSync;
+        private readonly IDisposable disposable;
+
+        private static readonly IReadOnlyDictionary<string, Viseme> phonomeMap
+            = new Dictionary<string, Viseme>
+            {
+                ["A"] = Viseme.aa,
+                ["I"] = Viseme.ih,
+                ["U"] = Viseme.ou,
+                ["E"] = Viseme.E,
+                ["O"] = Viseme.oh,
+            };
+
+        public ULipSyncAnimator(
+            FollowingLipAnimator followingLipAnimator,
+            global::uLipSync.uLipSync uLipSync,
+            float volumeThreshold = 0.01f,
+            bool verbose = false)
+        {
+            this.followingLipAnimator = followingLipAnimator;
+            this.uLipSync = uLipSync;
+
+            this.disposable = this.uLipSync
+                .ObserveEveryValueChanged(lipSync => lipSync.result)
+                .Subscribe(info =>
+                {
+                    if (verbose)
+                    {
+                        Debug.Log($"[FacialExpressions.uLipSync] Update lip sync: {info.phoneme}, {info.volume}");
+                    }
+
+                    if (phonomeMap.TryGetValue(info.phoneme, out var viseme) && info.volume > volumeThreshold)
+                    {
+                        SetTarget(new LipSample(viseme, weight: 1f));
+                    }
+                    else
+                    {
+                        SetDefault();
+                    }
+                });
+        }
+
+        public void Dispose()
+        {
+            Reset();
+            disposable.Dispose();
+        }
+
+        public void Update()
+        {
+            followingLipAnimator.Update();
+        }
+
+        public void Reset()
+        {
+            followingLipAnimator.Reset();
+        }
+
+        private void SetTarget(LipSample sample)
+        {
+            foreach (var viseme in phonomeMap.Values)
+            {
+                if (viseme == sample.viseme)
+                {
+                    followingLipAnimator.SetTarget(sample);
+                }
+                else
+                {
+                    followingLipAnimator.SetTarget(new LipSample(viseme, weight: 0f));
+                }
+            }
+        }
+
+        private void SetDefault()
+        {
+            foreach (var viseme in phonomeMap.Values)
+            {
+                followingLipAnimator.SetTarget(new LipSample(viseme, weight: 0f));
+            }
+        }
+    }
+}

Assets/Mochineko/FacialExpressions.Extensions/uLipSync/package.json

@@ -0,0 +1,11 @@
+{
+  "name": "com.mochineko.facial-expressions.extensions.ulipsync",
+  "version": "0.3.3",
+  "displayName": "Facial Expressions / uLipSync",
+  "description": "Facial expressions extension for uLipSync.",
+  "unity": "2021.3",
+  "author": {
+    "name": "Mochineko",
+    "email": "t.o.e.4315@gmail.com"
+  }
+}

Assets/Mochineko/FacialExpressions.Samples/Mochineko.FacialExpressions.Samples.asmdef

@@ -7,13 +7,15 @@
         "GUID:7ad15df7b990f4a14b759886eaecd6b9",
         "GUID:6d2977ee2adc849758706dc83e73429e",
         "GUID:b737d411f9698451ca2fe478a9e56123",
+        "GUID:a5d7cef0dd6fc404fb8a26bf152d9108",
         "GUID:83c9a70cfc06f4cc983a8a2efb97365f",
         "GUID:51d11816546934d4a937e4a420478ce5",
         "GUID:2620e54ab6edb48f48d55966fd5662fb",
         "GUID:ed5edd2439f854f75a164ed9d6e1f3bc",
         "GUID:e47c917724578cc43b5506c17a27e9a0",
         "GUID:8d76e605759c3f64a957d63ef96ada7c",
         "GUID:da3e51d19d51a544fa14d43fee843098",
+        "GUID:d444a9c79d21fec4181e6e12dc2e9706",
         "GUID:f51ebe6a0ceec4240a699833d6309b23"
     ],
     "includePlatforms": [],

Assets/Mochineko/FacialExpressions.Samples/SampleForULipSyncAndVRM.cs

@@ -0,0 +1,109 @@
+#nullable enable
+using System;
+using System.IO;
+using System.Threading;
+using Cysharp.Threading.Tasks;
+using Mochineko.FacialExpressions.Blink;
+using Mochineko.FacialExpressions.Emotion;
+using Mochineko.FacialExpressions.Extensions.uLipSync;
+using Mochineko.FacialExpressions.Extensions.VRM;
+using Mochineko.FacialExpressions.LipSync;
+using Mochineko.VOICEVOX_API;
+using UnityEngine;
+using UniVRM10;
+using VRMShaders;
+
+namespace Mochineko.FacialExpressions.Samples
+{
+    internal sealed class SampleForULipSyncAndVRM : MonoBehaviour
+    {
+        [SerializeField] private string path = string.Empty;
+        [SerializeField] private BasicEmotion basicEmotion = BasicEmotion.Neutral;
+        [SerializeField] private float emotionWeight = 1f;
+        [SerializeField] private float emotionFollowingTime = 1f;
+        [SerializeField] private uLipSync.uLipSync? uLipSync = null;
+
+        private ULipSyncAnimator? lipAnimator;
+        private IEyelidAnimator? eyelidAnimator;
+        private ExclusiveFollowingEmotionAnimator<BasicEmotion>? emotionAnimator;
+
+        private async void Start()
+        {
+            if (uLipSync == null)
+            {
+                throw new NullReferenceException(nameof(uLipSync));
+            }
+
+            VoiceVoxBaseURL.BaseURL = "http://127.0.0.1:50021";
+
+            var binary = await File.ReadAllBytesAsync(
+                path,
+                this.GetCancellationTokenOnDestroy());
+
+            var instance = await LoadVRMAsync(
+                binary,
+                this.GetCancellationTokenOnDestroy());
+
+            var lipMorpher = new VRMLipMorpher(instance.Runtime.Expression);
+            var followingLipAnimator = new FollowingLipAnimator(lipMorpher, followingTime: 0.15f);
+            lipAnimator = new ULipSyncAnimator(followingLipAnimator, uLipSync);
+
+            var eyelidMorpher = new VRMEyelidMorpher(instance.Runtime.Expression);
+            eyelidAnimator = new SequentialEyelidAnimator(eyelidMorpher);
+
+            var eyelidFrames = ProbabilisticEyelidAnimationGenerator.Generate(
+                Eyelid.Both,
+                blinkCount: 20);
+
+            eyelidAnimator.AnimateAsync(
+                    eyelidFrames,
+                    loop: true,
+                    this.GetCancellationTokenOnDestroy())
+                .Forget();
+
+            var emotionMorpher = new VRMEmotionMorpher(instance.Runtime.Expression);
+            emotionAnimator = new ExclusiveFollowingEmotionAnimator<BasicEmotion>(
+                emotionMorpher,
+                followingTime: emotionFollowingTime);
+        }
+
+        private void Update()
+        {
+            lipAnimator?.Update();
+            eyelidAnimator?.Update();
+            emotionAnimator?.Update();
+        }
+
+        private void OnDestroy()
+        {
+            lipAnimator?.Dispose();
+        }
+
+        private static async UniTask<Vrm10Instance> LoadVRMAsync(
+            byte[] binaryData,
+            CancellationToken cancellationToken)
+        {
+            cancellationToken.ThrowIfCancellationRequested();
+
+            return await Vrm10.LoadBytesAsync(
+                bytes: binaryData,
+                canLoadVrm0X: true,
+                controlRigGenerationOption: ControlRigGenerationOption.None,
+                showMeshes: true,
+                awaitCaller: new RuntimeOnlyAwaitCaller(),
+                materialGenerator: null,
+                vrmMetaInformationCallback: null,
+                ct: cancellationToken
+            );
+        }
+
+        [ContextMenu(nameof(Emote))]
+        public void Emote()
+        {
+            emotionAnimator?
+                .Emote(new EmotionSample<BasicEmotion>(
+                    basicEmotion,
+                    weight: emotionWeight));
+        }
+    }
+}