Skip to content

Commit

Permalink
v1.0.9 update OpenCVForUnity version to 2.6.4. update DlibFaceLandmar…
Browse files Browse the repository at this point in the history 
…kDetector version to 1.4.1.
  • Loading branch information
@EnoxSoftware
EnoxSoftware committed Nov 25, 2024
1 parent 6610dd8 commit 495e0bb
Show file tree
Hide file tree
Showing 17 changed files with 3,064 additions and 961 deletions.
126 changes: 78 additions & 48 deletions ...TK2/Assets/HoloLensWithDlibFaceLandmarkDetectorExample/HLARHeadExample/HLARHeadExample.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ namespace HoloLensWithDlibFaceLandmarkDetectorExample
/// HoloLens AR Head Example
/// An example of AR head projection using OpenCVForUnity and DlibLandmarkDetector on Hololens.
/// </summary>
[RequireComponent(typeof(HLCameraStreamToMatHelper), typeof(ImageOptimizationHelper))]
[RequireComponent(typeof(HLCameraStream2MatHelper), typeof(ImageOptimizationHelper))]
public class HLARHeadExample : MonoBehaviour
{
[SerializeField, HeaderAttribute("Preview")]
Expand Down Expand Up @@ -265,7 +265,7 @@ public class HLARHeadExample : MonoBehaviour
/// <summary>
/// The webcam texture to mat helper.
/// </summary>
HLCameraStreamToMatHelper webCamTextureToMatHelper;
HLCameraStream2MatHelper webCamTextureToMatHelper;

/// <summary>
/// The image optimization helper.
Expand Down Expand Up @@ -297,11 +297,6 @@ public class HLARHeadExample : MonoBehaviour
/// </summary>
string dlibShapePredictorFileName = "DlibFaceLandmarkDetector/sp_human_face_68.dat";

/// <summary>
/// The dlib shape predictor file path.
/// </summary>
string dlibShapePredictorFilePath;

Scalar COLOR_WHITE = new Scalar(255, 255, 255, 255);
Scalar COLOR_GRAY = new Scalar(128, 128, 128, 255);

Expand Down Expand Up @@ -386,8 +381,18 @@ bool isDetectingInFrameArrivedThread
public Text debugStr;


string cascade_filepath;
string cascade4Thread_filepath;
string dlibShapePredictor_filepath;
string dlibShapePredictor4Thread_filepath;

/// <summary>
/// The CancellationTokenSource.
/// </summary>
CancellationTokenSource cts = new CancellationTokenSource();

// Use this for initialization
protected void Start()
async void Start()
{
displayCameraPreviewToggle.isOn = displayCameraPreview;
enableDownScaleToggle.isOn = enableDownScale;
Expand All @@ -400,29 +405,69 @@ protected void Start()
enableLerpFilterToggle.isOn = enableLerpFilter;

imageOptimizationHelper = gameObject.GetComponent<ImageOptimizationHelper>();
webCamTextureToMatHelper = gameObject.GetComponent<HLCameraStreamToMatHelper>();
webCamTextureToMatHelper = gameObject.GetComponent<HLCameraStream2MatHelper>();
#if WINDOWS_UWP && !DISABLE_HOLOLENSCAMSTREAM_API
webCamTextureToMatHelper.frameMatAcquired += OnFrameMatAcquired;
#endif
webCamTextureToMatHelper.outputColorFormat = WebCamTextureToMatHelper.ColorFormat.GRAY;
webCamTextureToMatHelper.Initialize();

rectangleTracker = new RectangleTracker();


// Asynchronously retrieves the readable file path from the StreamingAssets directory.
if (debugStr != null)
{
debugStr.text = "Preparing file access...";
}

cascade_filepath = await DlibFaceLandmarkDetector.UnityUtils.Utils.getFilePathAsyncTask("OpenCVForUnity/objdetect/lbpcascade_frontalface.xml", cancellationToken: cts.Token);
//cascade4Thread_filepath = await DlibFaceLandmarkDetector.UnityUtils.Utils.getFilePathAsyncTask("OpenCVForUnity/objdetect/haarcascade_frontalface_alt.xml", cancellationToken: cts.Token);
cascade4Thread_filepath = await DlibFaceLandmarkDetector.UnityUtils.Utils.getFilePathAsyncTask("OpenCVForUnity/objdetect/lbpcascade_frontalface.xml", cancellationToken: cts.Token);
dlibShapePredictorFileName = HoloLensWithDlibFaceLandmarkDetectorExample.dlibShapePredictorFileName;
dlibShapePredictorFilePath = DlibFaceLandmarkDetector.UnityUtils.Utils.getFilePath(dlibShapePredictorFileName);
if (string.IsNullOrEmpty(dlibShapePredictorFilePath))
dlibShapePredictor_filepath = await DlibFaceLandmarkDetector.UnityUtils.Utils.getFilePathAsyncTask(dlibShapePredictorFileName, cancellationToken: cts.Token);
dlibShapePredictor4Thread_filepath = await DlibFaceLandmarkDetector.UnityUtils.Utils.getFilePathAsyncTask("DlibFaceLandmarkDetector/sp_human_face_6.dat", cancellationToken: cts.Token);

if (debugStr != null)
{
debugStr.text = "";
}

Run();
}

// Use this for initialization
void Run()
{
cascade = new CascadeClassifier();
cascade.load(cascade_filepath);
#if !WINDOWS_UWP || UNITY_EDITOR
// "empty" method is not working on the UWP platform.
if (cascade.empty())
{
Debug.LogError("cascade file is not loaded. Please copy from “OpenCVForUnity/StreamingAssets/OpenCVForUnity/objdetect/” to “Assets/StreamingAssets/OpenCVForUnity/objdetect/” folder. ");
}
#endif

cascade4Thread = new CascadeClassifier();
cascade4Thread.load(cascade4Thread_filepath);
#if !WINDOWS_UWP || UNITY_EDITOR
// "empty" method is not working on the UWP platform.
if (cascade4Thread.empty())
{
Debug.LogError("cascade file is not loaded. Please copy from “OpenCVForUnity/StreamingAssets/OpenCVForUnity/objdetect/” to “Assets/StreamingAssets/OpenCVForUnity/objdetect/” folder. ");
}
#endif

if (string.IsNullOrEmpty(dlibShapePredictor_filepath))
{
Debug.LogError("shape predictor file does not exist. Please copy from “DlibFaceLandmarkDetector/StreamingAssets/DlibFaceLandmarkDetector/” to “Assets/StreamingAssets/DlibFaceLandmarkDetector/” folder. ");
}
faceLandmarkDetector = new FaceLandmarkDetector(dlibShapePredictorFilePath);
faceLandmarkDetector = new FaceLandmarkDetector(dlibShapePredictor_filepath);

dlibShapePredictorFilePath = DlibFaceLandmarkDetector.UnityUtils.Utils.getFilePath("DlibFaceLandmarkDetector/sp_human_face_6.dat");
if (string.IsNullOrEmpty(dlibShapePredictorFilePath))
if (string.IsNullOrEmpty(dlibShapePredictor4Thread_filepath))
{
Debug.LogError("shape predictor file does not exist. Please copy from “DlibFaceLandmarkDetector/StreamingAssets/DlibFaceLandmarkDetector/” to “Assets/StreamingAssets/DlibFaceLandmarkDetector/” folder. ");
}
faceLandmarkDetector4Thread = new FaceLandmarkDetector(dlibShapePredictorFilePath);
faceLandmarkDetector4Thread = new FaceLandmarkDetector(dlibShapePredictor4Thread_filepath);


// set 3d face object points. (right-handed coordinates system)
Expand Down Expand Up @@ -469,6 +514,9 @@ protected void Start()

opticalFlowFilter = new OFPointsFilter((int)faceLandmarkDetector.GetShapePredictorNumParts());
opticalFlowFilter.diffCheckSensitivity /= imageOptimizationHelper.downscaleRatio;

webCamTextureToMatHelper.outputColorFormat = Source2MatHelperColorFormat.GRAY;
webCamTextureToMatHelper.Initialize();
}

/// <summary>
Expand Down Expand Up @@ -584,29 +632,7 @@ public void OnWebCamTextureToMatHelperInitialized()

mouthParticleSystem = mouth.GetComponentsInChildren<ParticleSystem>(true);


//grayMat = new Mat();
cascade = new CascadeClassifier();
cascade.load(Utils.getFilePath("OpenCVForUnity/objdetect/lbpcascade_frontalface.xml"));
#if !WINDOWS_UWP || UNITY_EDITOR
// "empty" method is not working on the UWP platform.
if (cascade.empty())
{
Debug.LogError("cascade file is not loaded. Please copy from “OpenCVForUnity/StreamingAssets/OpenCVForUnity/objdetect/” to “Assets/StreamingAssets/OpenCVForUnity/objdetect/” folder. ");
}
#endif

grayMat4Thread = new Mat();
cascade4Thread = new CascadeClassifier();
//cascade4Thread.load(Utils.getFilePath("OpenCVForUnity/objdetect/haarcascade_frontalface_alt.xml"));
cascade4Thread.load(Utils.getFilePath("OpenCVForUnity/objdetect/lbpcascade_frontalface.xml"));
#if !WINDOWS_UWP || UNITY_EDITOR
// "empty" method is not working on the UWP platform.
if (cascade4Thread.empty())
{
Debug.LogError("cascade file is not loaded. Please copy from “OpenCVForUnity/StreamingAssets/OpenCVForUnity/objdetect/” to “Assets/StreamingAssets/OpenCVForUnity/objdetect/” folder. ");
}
#endif
}

/// <summary>
Expand All @@ -630,15 +656,9 @@ public void OnWebCamTextureToMatHelperDisposed()
ExecuteOnMainThread.Clear();
}

if (cascade != null)
cascade.Dispose();

if (grayMat4Thread != null)
grayMat4Thread.Dispose();

if (cascade4Thread != null)
cascade4Thread.Dispose();

rectangleTracker.Reset();

camMatrix.Dispose();
Expand Down Expand Up @@ -667,12 +687,13 @@ public void OnWebCamTextureToMatHelperDisposed()
}

/// <summary>
/// Raises the web cam texture to mat helper error occurred event.
/// Raises the webcam texture to mat helper error occurred event.
/// </summary>
/// <param name="errorCode">Error code.</param>
public void OnWebCamTextureToMatHelperErrorOccurred(WebCamTextureToMatHelper.ErrorCode errorCode)
/// <param name="message">Message.</param>
public void OnWebCamTextureToMatHelperErrorOccurred(Source2MatHelperErrorCode errorCode, string message)
{
Debug.Log("OnWebCamTextureToMatHelperErrorOccurred " + errorCode);
Debug.Log("OnWebCamTextureToMatHelperErrorOccurred " + errorCode + ":" + message);
}

#if WINDOWS_UWP && !DISABLE_HOLOLENSCAMSTREAM_API
Expand Down Expand Up @@ -1536,6 +1557,12 @@ void OnDestroy()
webCamTextureToMatHelper.Dispose();
imageOptimizationHelper.Dispose();

if (cascade != null)
cascade.Dispose();

if (cascade4Thread != null)
cascade4Thread.Dispose();

if (faceLandmarkDetector != null)
faceLandmarkDetector.Dispose();

Expand All @@ -1544,6 +1571,9 @@ void OnDestroy()

if (rectangleTracker != null)
rectangleTracker.Dispose();

if (cts != null)
cts.Dispose();
}

/// <summary>
Expand Down
140 changes: 71 additions & 69 deletions .../Assets/HoloLensWithDlibFaceLandmarkDetectorExample/HLARHeadExample/HLARHeadExample.unity
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3393,8 +3393,8 @@ GameObject:
serializedVersion: 6
m_Component:
- component: {fileID: 1076083699}
- component: {fileID: 1076083697}
- component: {fileID: 1076083695}
- component: {fileID: 1076083701}
- component: {fileID: 1076083700}
m_Layer: 0
m_Name: HLARHeadExample
Expand All @@ -3417,74 +3417,6 @@ MonoBehaviour:
m_EditorClassIdentifier:
_downscaleRatio: 2
_frameSkippingRatio: 1
--- !u!114 &1076083697
MonoBehaviour:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 1076083694}
m_Enabled: 1
m_EditorHideFlags: 0
m_Script: {fileID: 11500000, guid: 0195895dd83eb204da131e4a7c4bcfe3, type: 3}
m_Name:
m_EditorClassIdentifier:
_requestedDeviceName:
_requestedWidth: 896
_requestedHeight: 504
_requestedIsFrontFacing: 0
_requestedFPS: 30
_rotate90Degree: 0
_flipVertical: 0
_flipHorizontal: 0
_outputColorFormat: 0
_timeoutFrameCount: 300
onInitialized:
m_PersistentCalls:
m_Calls:
- m_Target: {fileID: 1076083700}
m_TargetAssemblyTypeName:
m_MethodName: OnWebCamTextureToMatHelperInitialized
m_Mode: 1
m_Arguments:
m_ObjectArgument: {fileID: 0}
m_ObjectArgumentAssemblyTypeName: UnityEngine.Object, UnityEngine
m_IntArgument: 0
m_FloatArgument: 0
m_StringArgument:
m_BoolArgument: 0
m_CallState: 2
onDisposed:
m_PersistentCalls:
m_Calls:
- m_Target: {fileID: 1076083700}
m_TargetAssemblyTypeName:
m_MethodName: OnWebCamTextureToMatHelperDisposed
m_Mode: 1
m_Arguments:
m_ObjectArgument: {fileID: 0}
m_ObjectArgumentAssemblyTypeName: UnityEngine.Object, UnityEngine
m_IntArgument: 0
m_FloatArgument: 0
m_StringArgument:
m_BoolArgument: 0
m_CallState: 2
onErrorOccurred:
m_PersistentCalls:
m_Calls:
- m_Target: {fileID: 1076083700}
m_TargetAssemblyTypeName:
m_MethodName: OnWebCamTextureToMatHelperErrorOccurred
m_Mode: 0
m_Arguments:
m_ObjectArgument: {fileID: 0}
m_ObjectArgumentAssemblyTypeName: UnityEngine.Object, UnityEngine
m_IntArgument: 0
m_FloatArgument: 0
m_StringArgument:
m_BoolArgument: 0
m_CallState: 2
avoidAndroidFrontCameraLowLightIssue: 0
--- !u!4 &1076083699
Transform:
m_ObjectHideFlags: 0
Expand Down Expand Up @@ -3545,6 +3477,76 @@ MonoBehaviour:
videoFPS: {fileID: 443524926926304178}
trackFPS: {fileID: 443524928155071722}
debugStr: {fileID: 443524927024673313}
--- !u!114 &1076083701
MonoBehaviour:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 1076083694}
m_Enabled: 1
m_EditorHideFlags: 0
m_Script: {fileID: 11500000, guid: a80f11196f5e6394aaec3da665ebfc5a, type: 3}
m_Name:
m_EditorClassIdentifier:
_requestedDeviceName:
_requestedWidth: 896
_requestedHeight: 504
_requestedIsFrontFacing: 0
_requestedFPS: 30
_rotate90Degree: 0
_flipVertical: 0
_flipHorizontal: 0
_outputColorFormat: 3
_timeoutFrameCount: 1500
_onInitialized:
m_PersistentCalls:
m_Calls:
- m_Target: {fileID: 1076083700}
m_TargetAssemblyTypeName: HoloLensWithDlibFaceLandmarkDetectorExample.HLARHeadExample,
Assembly-CSharp
m_MethodName: OnWebCamTextureToMatHelperInitialized
m_Mode: 1
m_Arguments:
m_ObjectArgument: {fileID: 0}
m_ObjectArgumentAssemblyTypeName: UnityEngine.Object, UnityEngine
m_IntArgument: 0
m_FloatArgument: 0
m_StringArgument:
m_BoolArgument: 0
m_CallState: 2
_onDisposed:
m_PersistentCalls:
m_Calls:
- m_Target: {fileID: 1076083700}
m_TargetAssemblyTypeName: HoloLensWithDlibFaceLandmarkDetectorExample.HLARHeadExample,
Assembly-CSharp
m_MethodName: OnWebCamTextureToMatHelperDisposed
m_Mode: 1
m_Arguments:
m_ObjectArgument: {fileID: 0}
m_ObjectArgumentAssemblyTypeName: UnityEngine.Object, UnityEngine
m_IntArgument: 0
m_FloatArgument: 0
m_StringArgument:
m_BoolArgument: 0
m_CallState: 2
_onErrorOccurred:
m_PersistentCalls:
m_Calls:
- m_Target: {fileID: 1076083700}
m_TargetAssemblyTypeName: HoloLensWithDlibFaceLandmarkDetectorExample.HLARHeadExample,
Assembly-CSharp
m_MethodName: OnWebCamTextureToMatHelperErrorOccurred
m_Mode: 0
m_Arguments:
m_ObjectArgument: {fileID: 0}
m_ObjectArgumentAssemblyTypeName: UnityEngine.Object, UnityEngine
m_IntArgument: 0
m_FloatArgument: 0
m_StringArgument:
m_BoolArgument: 0
m_CallState: 2
--- !u!1 &1108638276
GameObject:
m_ObjectHideFlags: 0
Expand Down
Loading

0 comments on commit 495e0bb

Please sign in to comment.