diff --git a/README.md b/README.md index 43c1878..28116b8 100644 --- a/README.md +++ b/README.md @@ -33,4 +33,4 @@ The remap asset **must** be set in the **Anim blueprint**, in the **ZED Livelink For example, if you are using the Body format *Body_38*, open the **ABP_ZED_Manny** anim blueprint, select the **ZED LivelinkPose** component and, in the Detail panel, set the **Remap Asset** field to **RemapAssetBody38** -![](UnrealProject/images/remap_asset_selection.PNG) \ No newline at end of file +![](./images/remap_asset_selection.PNG) \ No newline at end of file diff --git a/UnrealProject/Content/Maps/LiveLinkMap.umap b/UnrealProject/Content/Maps/LiveLinkMap.umap index e012376..4a82643 100644 Binary files a/UnrealProject/Content/Maps/LiveLinkMap.umap and b/UnrealProject/Content/Maps/LiveLinkMap.umap differ diff --git a/UnrealProject/Content/ZED/Assets/Mannequin/Mesh/IKR_ZEDManny_Manny.uasset b/UnrealProject/Content/ZED/Assets/Mannequin/Mesh/IKR_ZEDManny_Manny.uasset new file mode 100644 index 0000000..525b6fc Binary files /dev/null and b/UnrealProject/Content/ZED/Assets/Mannequin/Mesh/IKR_ZEDManny_Manny.uasset differ diff --git a/UnrealProject/Content/ZED/Assets/Mannequin/Mesh/Manny/IK_Mannequin.uasset b/UnrealProject/Content/ZED/Assets/Mannequin/Mesh/Manny/IK_Mannequin.uasset new file mode 100644 index 0000000..dac2bfc Binary files /dev/null and b/UnrealProject/Content/ZED/Assets/Mannequin/Mesh/Manny/IK_Mannequin.uasset differ diff --git a/UnrealProject/Content/ZED/Assets/Mannequin/Mesh/Manny/PA_Mannequin.uasset b/UnrealProject/Content/ZED/Assets/Mannequin/Mesh/Manny/PA_Mannequin.uasset new file mode 100644 index 0000000..c553452 Binary files /dev/null and b/UnrealProject/Content/ZED/Assets/Mannequin/Mesh/Manny/PA_Mannequin.uasset differ diff --git a/UnrealProject/Content/ZED/Assets/Mannequin/Mesh/Manny/SKM_Manny_Simple.uasset b/UnrealProject/Content/ZED/Assets/Mannequin/Mesh/Manny/SKM_Manny_Simple.uasset new file mode 100644 index 0000000..d42525c Binary files /dev/null and b/UnrealProject/Content/ZED/Assets/Mannequin/Mesh/Manny/SKM_Manny_Simple.uasset differ diff --git a/UnrealProject/Content/ZED/Assets/Mannequin/Mesh/Manny/SK_Mannequin.uasset b/UnrealProject/Content/ZED/Assets/Mannequin/Mesh/Manny/SK_Mannequin.uasset new file mode 100644 index 0000000..41663b2 Binary files /dev/null and b/UnrealProject/Content/ZED/Assets/Mannequin/Mesh/Manny/SK_Mannequin.uasset differ diff --git a/UnrealProject/Content/ZED/Assets/Mannequin/Mesh/ZED_Manny/CR_ZED_Manny_FootIK.uasset b/UnrealProject/Content/ZED/Assets/Mannequin/Mesh/ZED_Manny/CR_ZED_Manny_FootIK.uasset new file mode 100644 index 0000000..be66292 Binary files /dev/null and b/UnrealProject/Content/ZED/Assets/Mannequin/Mesh/ZED_Manny/CR_ZED_Manny_FootIK.uasset differ diff --git a/UnrealProject/Content/ZED/Assets/Mannequin/Mesh/ZED_Manny/IKR_ZED_Manny.uasset b/UnrealProject/Content/ZED/Assets/Mannequin/Mesh/ZED_Manny/IKR_ZED_Manny.uasset index d3c828a..4ee1762 100644 Binary files a/UnrealProject/Content/ZED/Assets/Mannequin/Mesh/ZED_Manny/IKR_ZED_Manny.uasset and b/UnrealProject/Content/ZED/Assets/Mannequin/Mesh/ZED_Manny/IKR_ZED_Manny.uasset differ diff --git a/UnrealProject/Content/ZED/Assets/Mannequin/Mesh/ZED_Manny/PHYS_ZED_Manny.uasset b/UnrealProject/Content/ZED/Assets/Mannequin/Mesh/ZED_Manny/PHYS_ZED_Manny.uasset index bfb5c32..0ca167f 100644 Binary files a/UnrealProject/Content/ZED/Assets/Mannequin/Mesh/ZED_Manny/PHYS_ZED_Manny.uasset and b/UnrealProject/Content/ZED/Assets/Mannequin/Mesh/ZED_Manny/PHYS_ZED_Manny.uasset differ diff --git a/UnrealProject/Content/ZED/Assets/Mannequin/Mesh/ZED_Manny/SKM_ZED_Manny.uasset b/UnrealProject/Content/ZED/Assets/Mannequin/Mesh/ZED_Manny/SKM_ZED_Manny.uasset index ca04c84..7425255 100644 Binary files a/UnrealProject/Content/ZED/Assets/Mannequin/Mesh/ZED_Manny/SKM_ZED_Manny.uasset and b/UnrealProject/Content/ZED/Assets/Mannequin/Mesh/ZED_Manny/SKM_ZED_Manny.uasset differ diff --git a/UnrealProject/Content/ZED/Assets/Mannequin/Mesh/ZED_Manny/SK_ZED_Manny.uasset b/UnrealProject/Content/ZED/Assets/Mannequin/Mesh/ZED_Manny/SK_ZED_Manny.uasset index 543fcba..7e783e5 100644 Binary files a/UnrealProject/Content/ZED/Assets/Mannequin/Mesh/ZED_Manny/SK_ZED_Manny.uasset and b/UnrealProject/Content/ZED/Assets/Mannequin/Mesh/ZED_Manny/SK_ZED_Manny.uasset differ diff --git a/UnrealProject/Content/ZED/Blueprints/Animation/ABP_Manny_Simple.uasset b/UnrealProject/Content/ZED/Blueprints/Animation/ABP_Manny_Simple.uasset new file mode 100644 index 0000000..ddf7bef Binary files /dev/null and b/UnrealProject/Content/ZED/Blueprints/Animation/ABP_Manny_Simple.uasset differ diff --git a/UnrealProject/Content/ZED/Blueprints/Animation/ABP_ZED_Manny.uasset b/UnrealProject/Content/ZED/Blueprints/Animation/ABP_ZED_Manny.uasset index b0455ef..f0e3fc0 100644 Binary files a/UnrealProject/Content/ZED/Blueprints/Animation/ABP_ZED_Manny.uasset and b/UnrealProject/Content/ZED/Blueprints/Animation/ABP_ZED_Manny.uasset differ diff --git a/UnrealProject/Content/ZED/Blueprints/Animation/BP_Manny_Simple.uasset b/UnrealProject/Content/ZED/Blueprints/Animation/BP_Manny_Simple.uasset new file mode 100644 index 0000000..4467577 Binary files /dev/null and b/UnrealProject/Content/ZED/Blueprints/Animation/BP_Manny_Simple.uasset differ diff --git a/UnrealProject/Content/ZED/Blueprints/Animation/BP_ZEDLivelink_Manager.uasset b/UnrealProject/Content/ZED/Blueprints/Animation/BP_ZEDLivelink_Manager.uasset index a69f5c2..eb93beb 100644 Binary files a/UnrealProject/Content/ZED/Blueprints/Animation/BP_ZEDLivelink_Manager.uasset and b/UnrealProject/Content/ZED/Blueprints/Animation/BP_ZEDLivelink_Manager.uasset differ diff --git a/UnrealProject/Content/ZED/Blueprints/Animation/BP_ZED_Manny.uasset b/UnrealProject/Content/ZED/Blueprints/Animation/BP_ZED_Manny.uasset index f8450cf..78c03ee 100644 Binary files a/UnrealProject/Content/ZED/Blueprints/Animation/BP_ZED_Manny.uasset and b/UnrealProject/Content/ZED/Blueprints/Animation/BP_ZED_Manny.uasset differ diff --git a/UnrealProject/Content/ZED/Blueprints/Animation/CR_ZED_Manny_FootIK.uasset b/UnrealProject/Content/ZED/Blueprints/Animation/CR_ZED_Manny_FootIK.uasset index da558c2..2137574 100644 Binary files a/UnrealProject/Content/ZED/Blueprints/Animation/CR_ZED_Manny_FootIK.uasset and b/UnrealProject/Content/ZED/Blueprints/Animation/CR_ZED_Manny_FootIK.uasset differ diff --git a/UnrealProject/README.md b/UnrealProject/README.md deleted file mode 100644 index 5a43ddc..0000000 --- a/UnrealProject/README.md +++ /dev/null @@ -1,311 +0,0 @@ -# Getting started with Unreal Engine and our ZED LiveLink plugin - -We provide you an Unreal Project allowing to directly animate avatars with ZED LiveLink plugin skeleton data. It requires **Unreal Engine 4.26**. - -![](images/default_avatar_anim2.gif) - -This documentation aims to explain how joints informations (orientation and position) are brought by the ZED LiveLink plugin, and how to use them to animate an avatar directly in Unreal. - - -## ZED Live Link skeleton data: joint format - -This section will present the joint format output by the ZED LiveLink plugin. - -Live Link works with the notion of **Subject**, which is an individual stream of data, typically describing a given individual character. -For a given Live Link subject, at a given frame, we output a total of 32 joints, in the following order: - - -| Joint index | Joint name | -|-------------|------------| -| 0 | PELVIS | -| 1 | SPINE_NAVAL | -| 2 | SPINE_CHEST | -| 3 | NECK | -| 4 | CLAVICLE_LEFT | -| 5 | SHOULDER_LEFT | -| 6 | ELBOW_LEFT | -| 7 | WRIST_LEFT | -| 8 | HAND_LEFT | -| 9 | HANDTIP_LEFT | -| 10 | THUMB_LEFT | -| 11 | CLAVICLE_RIGHT | -| 12 | SHOULDER_RIGHT | -| 13 | ELBOW_RIGHT | -| 14 | WRIST_RIGHT | -| 15 | HAND_RIGHT | -| 16 | HANDTIP_RIGHT | -| 17 | THUMB_RIGHT | -| 18 | HIP_LEFT | -| 19 | KNEE_LEFT | -| 20 | ANKLE_LEFT | -| 21 | FOOT_LEFT | -| 22 | HIP_RIGHT | -| 23 | KNEE_RIGHT | -| 24 | ANKLE_RIGHT | -| 25 | FOOT_RIGHT | -| 26 | HEAD | -| 27 | NOSE | -| 28 | EYE_LEFT | -| 29 | EAR_LEFT | -| 30 | EYE_RIGHT | -| 31 | EAR_RIGHT | - - - -![](images/body_converted_32_joints.jpg) - -Our LiveLink plugin will feed local rotation and translation information for each of these joints. Local rotation for a joint is defined as its rotation relative to its parent joint. You can have an idea of the joint hierarchy on the previous image (a skeleton is basically a tree). We see that the root of this skeleton is the *PELVIS*, which has 3 children joints (*SPINE_NAVAL*, *HIP_LEFT* and *HIP_RIGHT*), etc. In addition to these local rotations (given in the form of a quaternion (x,y,z,w)) and translations, our LiveLink plugin will give the root global rotation and translation. - - -All these joints informations are expressed in Unreal coordinate frame, that is LEFT_HANDED_Z_UP. Each joint orientation will be expressed in the same coordinate frame you can see here for the avatar's hips: - -![](./images/coordinate_frame.jpg) - -## Necessary transformation of SDK raw data in Unreal - -To animate an avatar inside an Unreal project using raw SDK data fed by the ZED Live Link plugin, some operations must be done. The Unreal project we provide you already implements these transformations, and this section will explain how they are implemented. All these transformations are done using a **Remap Asset**. - -The idea is first to create a new C++ class in your project, and select *LiveLinkRemapAsset* as its parent class. - -![](images/create_remap_asset_class.jpg) - -In this class you'll be able to implement the following operations: - - -- Apply only Live Link rotations without modifying translations, except for the root. Indeed, Live Link default behaviour is to apply both translation and rotation data to the avatar, but you might prefer to apply only rotations and keep the local translation of the avatar to avoid deforming it. - - -- Change the root translation to adapt to your avatar's scale and make sure the avatar still has its feet on the floor. If an avatar is two times bigger than the skeleton output by the SDK, we will need to multiply the root translation by 2. There are two parameters to be handled here: first, the actual leg size of the avatar compared to the leg size of the skeleton coming from the SDK. You can get a first multiplication factor by checking these two sizes directly on the bones of the avatar. But the avatar might also have a scale not equal to 1. In this case, this scale won't actually alter the bone size of the avatar, so you need to take this scale into account in addition to the first factor. Consequently, the multiplication factor to apply to the root translation will be (avatarLegSize / SDKLegSize) * avatarScale. - - -- Apply extra rotations in case the avatar's T-Pose doesn't correspond to a situation where all its local orientations are null. - -Let's explain a bit more the last point. Raw SDK orientations fed by LiveLink are output to be applied on an avatar who is in T-Pose when all its local orientations are null. The problem is that a lot of avatars you can import in Unreal will have a reference pose that is a T-Pose as excepted, but won't necessarily correspond to null orientations. - -For instance, if you import this avatar from Mixamo, you'll see its reference pose is a T-Pose: - -![](images/michelle_tpose.jpg) - -But by putting all its local orientations to 0, the resulting pose will be this: - -![](images/michelle_null_orientations.jpg) - - In order to have an Unreal project that generalizes to a lot of avatars, it can be interesting to apply extra rotations to animate all avatars whose reference pose is a T-Pose. - -So the idea is first to apply the actual rotation that puts the avatar from its "null orientations" pose to its T-Pose. This rotation information can be retrieved using [FBaseCompactPose::GetRefPose](https://docs.unrealengine.com/4.26/en-US/API/Runtime/Engine/FBaseCompactPose/GetRefPose/). - -The problem is that after applying these rotations, the avatar is in T-Pose, but the rotated joints as well as their children won't be expressed in the same coordinate frame as before. For instance, you can see here that the avatar's left shoulder coordinate frame has been rotated with the same rotation applied to the joint: - -![](images/michelle_leftshoulder.jpg) - -The SDK orientations are expressed in *LEFT_HANDED_Z_UP* coordinate frame with the avatar facing the +X axis, so we can't directly apply the orientations on this new coordinate frame. To apply rotations correctly, we must do the following: - -- For each joint, apply the kinematic chain of all its parents joint rotation in order to put it in the right coordinate frame. For a given joint whose parent is the joint jn, we will apply R_j0 * R_j1 * ... R_jn where jn-1 is the parent of jn. -- Apply the SDK rotations, now that we are in the right coordinate frames. -- Apply the inverse kinematic chain computed in the first step to put the joints back in their coordinate frame. - -You can find in the Unreal Project we provide you an implementation of these transformations that is directly useable: *[LiveLinkOrientationsRemapAsset](Source/ZEDUnrealLiveLink/LiveLinkOrientationsRemapAsset.cpp)* - - - -Next, if your avatar bones don't have the same name than the ZED SDK bone names (explained in the first section), you need to create a new Blueprint class, which parent class must be the C++ **Remap asset** class you just created. - -![](images/createremapasset.jpg) - -Open this blueprint class and override the function "GetRemappedBoneNames". There you must do a similar remapping than on the picture below, where bones explained in the first section must be mapped to their name in your imported avatar. The bone names in the "Switch on Name" node correspond to the bone names given by our LiveLink plugin, and the bone names in the "Return Node" nodes correspond to your imported avatar bone names. The Unreal project we provided you has an example of such a Remap Asset useable for Mixamo avatars, called *MixamoRemap*. - -![](images/remappingasset.jpg) - -Once done, you must apply this **Remap Asset** to the Live Link data. Data from Live Link can be applied directly inside an Animation Blueprint, and when creating an Animation Blueprint for your avatar, you'll have to include a **Live Link Pose** node. This is where you can put the custom **Remap Asset** you created to transform the skeleton data coming from Live Link. - -![](images/animgraph.jpg) - -Implementing the above transformations will allow you to have a similar behaviour than with our provided Unreal project. In the rest of the documentation we'll assume you're using this provided Unreal project and we will explain how to directly animate an avatar and how to add new avatars to the project. - - -## Getting started with our Unreal Project - -Open our *ZEDUnrealLiveLink.uproject* project. You'll see the following empty scene: - -![](./images/project_screen.jpg) - - In the Content folder of the editor, you'll see that there are already 3 Animation Blueprints, that correspond to 3 different avatars already imported in the project, as well as 2 assets called *Avatar_livelink* and *MixamoRemap*. We'll see in the section *Add a new avatar and animate it with our plugin* what's their use and how you can add your own assets to animate new avatars. - - This project already handles the Live Link skeleton data in order to instantiate correctly avatars and place them in the scene with the corresponding joint positions and orientations. If you want to see how this is done, you can check the project **Level Blueprint**. - - -## Enabling the Live Link Plugin - -You can follow these steps if the Live Link Plugin is not enabled yet in your project. - -1. Inside your project, In the Menu Bar under **Edit**, select **Plugins**. - - -![](./images/capture_plugin.jpg) - - -2. Under the Animation section, click Enabled for Live Link, and Yes on the confirmation window, then restart the Editor. - - -![](./images/capture_livelink_install.jpg) - -## Using the ZED Live Link app - -At this point, you must launch the ZED Live Link plugin that will send skeleton data to the Unreal project. - -### With the Multicam API - -The plugin requires the reloc file generated by the ZED Locator tool to run. It specify the position of each camera used by the multicamera API. - -Once this file is generated : - -1. Go to \Engine\Binaries\Win64\ZEDLiveLink and open a terminal. -2. Run ./ZEDLiveLink with the path of the rloc file as an argument. - -For example, run : - -```bash -$ ./ZEDLiveLink C:/UnrealEngine/ZEDLiveLink/ZED_Locator.json -``` - -### With the classic api - -With the single camera API, the configuration file is not needed. -The input (ip, svo or serial number) can be provided directly as an argument. - -To send data from a specific camera, add its serial number as arguments. -For example, to connect the camera whose serial number is 10028418, run : - -```bash -$ ./ZEDLiveLink 10028418 -``` - -3. You can see the connection status in the terminal. - -- On Windows : - -![](../doc_images/capture_zed_connected.jpg) - -- On Linux : - -![](../doc_images/zed_capture_installed_linux.jpg) - - -## Connecting Live Link source to UE4 - -This step is normally handled automatically by the Unreal Project provided. If you have data coming from several Live Link sources, you might need to do the following to make sure you're connected to the right source. - -1. Inside the UE4 Editor, from the Menu Bar under **Window**, select **Live Link**. - - -![](./images/capture_menu_livelink.jpg) - - -2. In the Live Link window, click the **Source** button, then under Message Bus Source, select your ZED source. - - -![](./images/capture_livelink_zed.jpg) - - -## Animate default avatar - -With the project you're provided, you can animate a default avatar by directly hitting play in Unreal. You will see avatars animated with skeleton data coming from the connected Live Link source: - - -![](images/default_avatar_anim2.gif) - -## Add a new avatar and animate it with our plugin - -The Unreal project we provided you already contains avatars you can animate. This section will explain how to import a new avatar to make it work with our ZED LiveLink plugin. You can find various avatars on [Mixamo website](https://www.mixamo.com/) for instance. You should follow these steps: - - -### 1. Importing a new avatar -For now, our plugin allows to animate correctly an avatar if it is imported in the right format. We need the avatar to respect two conditions: - -- The first condition is on the avatar's reference pose. The reference pose is the the default pose of the avatar you can see when you open the skeleton asset window. We need this reference pose to be in T-Pose, which corresponds to the pose where the avatar is standing up with its arms held horizontally. - -- The second condition is on the avatar's global orientation when imported. We want to start from a known orientation, so we need the avatar to be facing the +X direction in its reference pose. - -To respect the first condition, you might need to change your avatar's reference pose if it isn't already a T-Pose. You can change its reference pose using **Blender**, for instance. Let's take the example of this robot avatar, that initally has an "A-Pose": - -![](images/robot_blender_apose.jpg) - -You can change its reference pose this way: - -1. Select the **Armature** of your avatar in the **Scene Collection** and select the **Pose Mode** this way: - -![](images/robot_blender_posemode.jpg) - -2. For each bone that needs to be rotated, click on it and go to **Pose/Transform/Rotate**: - -![](images/robot_blender_rotate.jpg) - -Once you did all the necessary rotations and that your avatar appears in T-Pose, you can define it as the new reference pose (which is called *Rest Pose* in **Blender**). Go to **Pose/Apply** and select **Apply Pose as Rest Pose**: - -![](images/robot_blender_tpose.jpg) - -You can then export your avatar as an FBX object and its reference pose will be the T-Pose you defined. - -When you import your avatar in Unreal, you'll have to tick the checkboxes *Update Skeleton Reference Pose* and *Use T0 As Ref Pose* to make sure the avatar's reference pose in Unreal is the one you just defined in Blender. - -Moreover, in order to meet the second condition, you'll have to tick the checkbox *Force Front X Axis* to force the avatar to look at the +X direction. This way we make sure the avatar starts with a known orientation. - -![](images/import_checkboxes.jpg) - -Make sure, by checking the avatar skeleton asset, that its reference pose is in T-Pose, and is looking at the +X direction. This should look like this: - -![](images/robot_unreal_tpose.png) - -### 2. Creating a Remap Asset - -Once you imported the avatar, the next thing to do is to create a **Remap Asset**. Indeed, your avatar joints probably don't have the same name than the one given by our plugin. It's for example the case with Mixamo avatars we used in the Unreal project you're provided. Live Link gives the possibility to remap these bone names. Create a new Blueprint class, which parent class must be "LiveLinkOrientationsRemapAsset". - -![](images/createremapasset.jpg) - -Open this blueprint class and override the function "GetRemappedBoneNames". There you must do a similar remapping than on the picture below, where bones explained in the first section must be mapped to their name in your imported avatar. The bone names in the "Switch on Name" node correspond to the bone names given by our LiveLink plugin, and the bone names in the "Return Node" nodes correspond to your imported avatar bone names. The Unreal project we provided you has an example of such a Remap Asset useable for Mixamo avatars, called *MixamoRemap*. - -![](images/remappingasset.jpg) - -### 3. Creating an Animation Blueprint - -Data from Live Link can be applied directly inside Animation Blueprints. - -When creating an Animation Blueprint, a dialog box will allow you to select its "Target Skeleton". Choose the skeleton created when you imported your avatar. - -![](images/createanimblueprint.jpg) - -This is achieved by creating a Live Link Pose node inside the AnimGraph of an Animation Blueprint: -- Create a variable **Live Link Subject** and put it in the AnimGraph. It refers to the name of the subject in LiveLink to stream data from. In case several persons are detected, this variable will feed the right skeleton data to the right avatar. -- Link it to a **Live Link Pose** node, which will get the actual skeleton data from our Live Link Subject. -- Link this node to the **Output Pose**, this way the Live Link skeleton data will directly animate our avatar. - -In the "Details" menu of the **Live Link Pose** node, you have the possibility to select a retarget asset. This is where you must select the Remap Asset you created in the previous step, in order to remap correctly the Live Link skeleton data to your avatar bone names. - -The AnimGraph should look like this: - -![](images/animgraph.jpg) - -You can refer to the existing Animation Blueprints in the Unreal project to configure yours the same way. - -### 4. Configuring "Avatar_livelink" asset - -Next, you must select the "Avatar_livelink" asset already provided in the Unreal Project. The role of this asset is to define which avatar will be animated in the Unreal scene. In the "Skeletal Mesh" details menu, you must select the Skeletal Mesh you imported and want to use, and the Anim Class linked to this mesh that you previously created. - -The Live Link Skeletal Animation component has be added to this Actor in order to drive its parameters with Live Link from a connected external source. - -![](images/avatarlivelink.jpg) - -### 5. Configuring project level blueprint - -Once the "Avatar_livelink" is set, last step is to open the Level Blueprint of the project. This Level Blueprint automatically connects the project to the Live Link Source at Play, and a Avatar_livelink Actor is instantiated for each skeleton sent by the LiveLink plugin, in order to be rendered on the Unreal scene. - -At the right side of the EventGraph, you will notice two nodes "Cast To Anim_livelink" and "SET Live Link Subject". - -![](images/levelblueprint1.jpg) - -The idea is to replace these nodes with a cast to your own Animation Blueprint you created for your avatar. Remove the two current nodes (or suppress their links to rest of the graph), and create a cast to your own Animation Blueprint, as well as a set to its "Live Link Subject" variable. Reproduce the same connections set before with your Animation Blueprint. - - -![](images/levelblueprint2.jpg) - -Once done, you're ready to press play and see your avatar animated with LiveLink data. diff --git a/UnrealProject/Source/ZEDUnrealLiveLink.Target.cs b/UnrealProject/Source/ZEDUnrealLiveLink.Target.cs index 529d0f2..80e0e1c 100644 --- a/UnrealProject/Source/ZEDUnrealLiveLink.Target.cs +++ b/UnrealProject/Source/ZEDUnrealLiveLink.Target.cs @@ -10,8 +10,6 @@ public ZEDUnrealLiveLinkTarget(TargetInfo Target) : base(Target) Type = TargetType.Game; DefaultBuildSettings = BuildSettingsVersion.V2; - BuildEnvironment = TargetBuildEnvironment.Unique; - GlobalDefinitions.Add("ALLOW_UDP_MESSAGING_SHIPPING=1"); ExtraModuleNames.AddRange( new string[] { "ZEDUnrealLiveLink" } ); diff --git a/UnrealProject/ZEDUnrealLiveLink.uproject b/UnrealProject/ZEDUnrealLiveLink.uproject index ffdd4b2..a2dc67a 100644 --- a/UnrealProject/ZEDUnrealLiveLink.uproject +++ b/UnrealProject/ZEDUnrealLiveLink.uproject @@ -1,6 +1,6 @@ { "FileVersion": 3, - "EngineAssociation": "{450BA52B-48CB-098E-7C7F-69A0E7DEE36E}", + "EngineAssociation": "5.1", "Category": "", "Description": "", "Modules": [ diff --git a/UnrealProject/images/.gitkeep b/UnrealProject/images/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/UnrealProject/images/animgraph.jpg b/UnrealProject/images/animgraph.jpg deleted file mode 100644 index e660373..0000000 Binary files a/UnrealProject/images/animgraph.jpg and /dev/null differ diff --git a/UnrealProject/images/avatarlivelink.jpg b/UnrealProject/images/avatarlivelink.jpg deleted file mode 100644 index 13d73ea..0000000 Binary files a/UnrealProject/images/avatarlivelink.jpg and /dev/null differ diff --git a/UnrealProject/images/body_converted_32_joints.jpg b/UnrealProject/images/body_converted_32_joints.jpg deleted file mode 100644 index b643a39..0000000 Binary files a/UnrealProject/images/body_converted_32_joints.jpg and /dev/null differ diff --git a/UnrealProject/images/capture_livelink_install.jpg b/UnrealProject/images/capture_livelink_install.jpg deleted file mode 100644 index 9f777f6..0000000 Binary files a/UnrealProject/images/capture_livelink_install.jpg and /dev/null differ diff --git a/UnrealProject/images/capture_plugin.jpg b/UnrealProject/images/capture_plugin.jpg deleted file mode 100644 index 086fd93..0000000 Binary files a/UnrealProject/images/capture_plugin.jpg and /dev/null differ diff --git a/UnrealProject/images/coordinate_frame.jpg b/UnrealProject/images/coordinate_frame.jpg deleted file mode 100644 index 5e489b1..0000000 Binary files a/UnrealProject/images/coordinate_frame.jpg and /dev/null differ diff --git a/UnrealProject/images/create_remap_asset_class.jpg b/UnrealProject/images/create_remap_asset_class.jpg deleted file mode 100644 index 50f15a6..0000000 Binary files a/UnrealProject/images/create_remap_asset_class.jpg and /dev/null differ diff --git a/UnrealProject/images/createanimblueprint.jpg b/UnrealProject/images/createanimblueprint.jpg deleted file mode 100644 index d76659d..0000000 Binary files a/UnrealProject/images/createanimblueprint.jpg and /dev/null differ diff --git a/UnrealProject/images/createremapasset.jpg b/UnrealProject/images/createremapasset.jpg deleted file mode 100644 index d2a75aa..0000000 Binary files a/UnrealProject/images/createremapasset.jpg and /dev/null differ diff --git a/UnrealProject/images/forcefrontxaxis.jpg b/UnrealProject/images/forcefrontxaxis.jpg deleted file mode 100644 index 6df9eaf..0000000 Binary files a/UnrealProject/images/forcefrontxaxis.jpg and /dev/null differ diff --git a/UnrealProject/images/import_checkboxes.jpg b/UnrealProject/images/import_checkboxes.jpg deleted file mode 100644 index 48933a4..0000000 Binary files a/UnrealProject/images/import_checkboxes.jpg and /dev/null differ diff --git a/UnrealProject/images/levelblueprint1.jpg b/UnrealProject/images/levelblueprint1.jpg deleted file mode 100644 index 3c07288..0000000 Binary files a/UnrealProject/images/levelblueprint1.jpg and /dev/null differ diff --git a/UnrealProject/images/levelblueprint2.jpg b/UnrealProject/images/levelblueprint2.jpg deleted file mode 100644 index 047f948..0000000 Binary files a/UnrealProject/images/levelblueprint2.jpg and /dev/null differ diff --git a/UnrealProject/images/michelle_leftshoulder.jpg b/UnrealProject/images/michelle_leftshoulder.jpg deleted file mode 100644 index 0fff3f4..0000000 Binary files a/UnrealProject/images/michelle_leftshoulder.jpg and /dev/null differ diff --git a/UnrealProject/images/michelle_null_orientations.jpg b/UnrealProject/images/michelle_null_orientations.jpg deleted file mode 100644 index b4adb0f..0000000 Binary files a/UnrealProject/images/michelle_null_orientations.jpg and /dev/null differ diff --git a/UnrealProject/images/michelle_tpose.jpg b/UnrealProject/images/michelle_tpose.jpg deleted file mode 100644 index a59b7c1..0000000 Binary files a/UnrealProject/images/michelle_tpose.jpg and /dev/null differ diff --git a/UnrealProject/images/project_screen.jpg b/UnrealProject/images/project_screen.jpg deleted file mode 100644 index 34672a7..0000000 Binary files a/UnrealProject/images/project_screen.jpg and /dev/null differ diff --git a/UnrealProject/images/remappingasset.jpg b/UnrealProject/images/remappingasset.jpg deleted file mode 100644 index 91b14cb..0000000 Binary files a/UnrealProject/images/remappingasset.jpg and /dev/null differ diff --git a/zed-livelink-fusion/doc_images/EnableByDefault.jpg b/images/EnableByDefault.jpg similarity index 100% rename from zed-livelink-fusion/doc_images/EnableByDefault.jpg rename to images/EnableByDefault.jpg diff --git a/zed-livelink-fusion/doc_images/capture_livelink_camera.jpg b/images/capture_livelink_camera.jpg similarity index 100% rename from zed-livelink-fusion/doc_images/capture_livelink_camera.jpg rename to images/capture_livelink_camera.jpg diff --git a/images/capture_livelink_install2.jpg b/images/capture_livelink_install2.jpg new file mode 100644 index 0000000..a01caba Binary files /dev/null and b/images/capture_livelink_install2.jpg differ diff --git a/UnrealProject/images/capture_livelink_zed.jpg b/images/capture_livelink_zed.jpg similarity index 100% rename from UnrealProject/images/capture_livelink_zed.jpg rename to images/capture_livelink_zed.jpg diff --git a/UnrealProject/images/capture_menu_livelink.jpg b/images/capture_menu_livelink.jpg similarity index 100% rename from UnrealProject/images/capture_menu_livelink.jpg rename to images/capture_menu_livelink.jpg diff --git a/images/capture_plugin2.jpg b/images/capture_plugin2.jpg new file mode 100644 index 0000000..94ab78e Binary files /dev/null and b/images/capture_plugin2.jpg differ diff --git a/zed-livelink-fusion/doc_images/capture_zed_addsubject.jpg b/images/capture_zed_addsubject.jpg similarity index 100% rename from zed-livelink-fusion/doc_images/capture_zed_addsubject.jpg rename to images/capture_zed_addsubject.jpg diff --git a/images/capture_zed_connected2.jpg b/images/capture_zed_connected2.jpg new file mode 100644 index 0000000..534c824 Binary files /dev/null and b/images/capture_zed_connected2.jpg differ diff --git a/zed-livelink-fusion/doc_images/capture_zed_subject_linux.jpg b/images/capture_zed_subject_linux.jpg similarity index 100% rename from zed-livelink-fusion/doc_images/capture_zed_subject_linux.jpg rename to images/capture_zed_subject_linux.jpg diff --git a/zed-livelink-fusion/doc_images/livelink_component.jpg b/images/livelink_component.jpg similarity index 100% rename from zed-livelink-fusion/doc_images/livelink_component.jpg rename to images/livelink_component.jpg diff --git a/zed-livelink-fusion/doc_images/livelink_component_subject.jpg b/images/livelink_component_subject.jpg similarity index 100% rename from zed-livelink-fusion/doc_images/livelink_component_subject.jpg rename to images/livelink_component_subject.jpg diff --git a/UnrealProject/images/remap_asset_selection.PNG b/images/remap_asset_selection.jpg similarity index 100% rename from UnrealProject/images/remap_asset_selection.PNG rename to images/remap_asset_selection.jpg diff --git a/zed-livelink-fusion/doc_images/zed_capture_installed_linux.jpg b/images/zed_capture_installed_linux.jpg similarity index 100% rename from zed-livelink-fusion/doc_images/zed_capture_installed_linux.jpg rename to images/zed_capture_installed_linux.jpg diff --git a/zed-livelink-fusion/README.md b/zed-livelink-fusion/README.md index 7957b9a..6da1c87 100644 --- a/zed-livelink-fusion/README.md +++ b/zed-livelink-fusion/README.md @@ -15,12 +15,20 @@ It can be send through local network on Windows and Linux computers. ## Getting started - First, download the latest version of the ZED SDK on [stereolabs.com](https://www.stereolabs.com/developers/) -- For more information, read the ZED [Documentation](https://www.stereolabs.com/docs) and [API documentation](https://www.stereolabs.com/docs/api/) +- For more information, read the ZED [Documentation](https://www.stereolabs.com/docs) and [live link documentation](https://www.stereolabs.com/docs/livelink/) -To compile the tool from source, you will require a source build of Unreal Engine. +To compile the tool from source, you will require a source build of Unreal Engine. For more information, read the [Building ZED Live Link Plugin](https://www.stereolabs.com/docs/livelink/building-the-plugin/) documentation page. -## Download the Engine +## Using ZED Live Link + +We provide you an Unreal project that already contains the different assets necessary to animate avatars properly using the **ZED Live Link** skeleton data. + +You will be able to use directly this project by hitting play while the **ZED Live Link sample** is running and see a default avatar animate. We are showing how to add your own models here: [Animate New Avatar](/livelink/animate-new-avatar/). + +In this tutorial, you'll learn how to use our **ZED Live Link sample**, connect it to our Unreal Project and animate a default avatar with the **Live Link** skeleton data. You must follow these steps: + +### 1. Download the Engine To gain access to the UE engine code, please follow the steps below: @@ -34,93 +42,108 @@ To gain access to the UE engine code, please follow the steps below: > *If the invitation isn't sent, clicking on the following link seems to trigger it: [Unreal Group](https://github.com/orgs/EpicGames). +### 2. Open the Unreal Project -## Build for Windows +Open our *ZEDUnrealLiveLink.uproject* project. -### Compile from Source +* In your project root directory right click on the **ZEDUnrealLiveLink.uproject** file and select **Generate Visual Studio project files** in the context menu. +* Open Visual Studio and compile. Launch the project from the **uproject** file or Visual Studio. -1. Inside the root directory, run **Setup.bat**. This will check all the project dependencies and update them as needed. -2. Clone this repository inside of \Engine\Source\Programs. -3. Run **GenerateProjectFiles.bat** to create project files for the engine. -4. Load the project into Visual Studio by double-clicking on the UE5.sln file, then right click on the ZEDLiveLink target and select **Build**. -5. The binaries will be placed in \Engine\Binaries\Win64\ZEDLiveLink\. +If these options are not available, make sure your Visual Studio install is correctly setup. -## Build for Linux + ### 3. Enable the Live Link Plugin in Unreal -### Compile from Source +You can follow these steps if the Live Link Plugin is not enabled yet in your project. -1. Go into the folder you just downloaded/cloned and run **Setup.sh** from the terminal. This will check all the project dependencies and update them as needed. -2. Clone this repository inside of /Engine/Source/Programs. -3. In the Engine root directory, run **GenerateProjectFiles.sh**. -4. Run **make** to build the Engine. -5. Go to UnrealEngine/Engine/Build/BatchFiles and open a terminal. -6. Build the plugin with the command : - ```bash - $ ./RunUAT.sh BuildGraph -Script=Engine/Source/Programs/zed-livelink-plugin/BuildZEDLinux.xml -Target="Stage ZEDLiveLink Linux" - ``` -7. The binaries will be placed in /Engine/Binaries/Linux/ZEDLiveLink/. +1. Inside your project, In the Menu Bar under **Edit**, select **Plugins**. + + +![](../images/capture_plugin2.jpg) + + +2. Under the Animation section, click Enabled for Live Link, and Yes on the confirmation window, then restart the Editor. + + +![](../images/capture_livelink_install2.jpg) + +### 4. Select the remap asset + +The ZED SDK now has multiple skeleton formats (Body 34, 38, and 70) available for animating a 3D model. +You need to make sure the correct remap asset is selected in the Anim Blueprint of Actor you are using in your level. Indeed, each body format has its own remap asset (as the name and numbers of joint is different). +The remap asset **must** be set in the **Anim blueprint**, in the **ZED Livelink** pose component. + +For example, if you are using the Body format *Body_38*, open the **ABP_ZED_Manny** anim blueprint, select the **ZED LivelinkPose** component and, in the Detail panel, set the **Remap Asset** field to **RemapAssetBody38** + +![](../images/remap_asset_selection.jpg) + +### 4. Run ZED Live Link + +The release 4.0 of the ZED SDK introduced the Fusion API which allow you to fuse skeleton data from multiples cameras. This features is also supported in the project. + +At this point, you must run the ZED Live Link executable. It will call the ZED SDK in order to compute skeleton data and stream it to the Unreal project. So you must make sure to have the ZED SDK installed on the machine that will run the ZED Live Link sample. + +A pre-built `ZEDLiveLinkFusion` sender for the current SDK version is located in `zed-livelink/zed-livelink-fusion/Releases/your system/`. +To learn how to build the ZED Live link sample yourself, take a look at our [Build Zed Live Link](/livelink/building-the-plugin/) documentation page. -### Using the Live Link app +- Open a terminal in the folder containing the ZED Live Link executable, and run -The ZED Live link sample now requires a Config file (Json format) to run. This file contains all the parameters from the ZED SDK that can be modified in this sample. +```bash +$ ./ZEDLiveLinkFusion +``` + +#### Adjust ZED SDK Parameters + +The ZED Live link sample requires a configuration file (Json format) to run. This file contains all the parameters from the ZED SDK that can be modified in this sample. By default, the sample will try to open a json called "ZEDFusionLiveLinkConfig.json" located next to this executable. The Fusion API requires a calibration file to be able to fuse the data from multiple camera. This file cam be generated using the ZED 360 tool available with the ZED SDK. Once your calibration file is ready, in the ZEDFusionLivelink.json file, set the value of **json_config_filename** to the path to this calibration file. -Your configuration file will look like this : +The configuration file looks like this : -```bash + ```json { "InitParameters": { - "resolution": "HD1080", - "fps": 30, - "depth_mode": "ULTRA" + "resolution": "HD1080", + "fps": 30, + "depth_mode": "ULTRA" }, "BodyTrackingParameters": { - "detection_model": "HUMAN_BODY_MEDIUM", - "body_format": "BODY_38", - "body_selection": "FULL", - "confidence": 15, - "max_range": -1, - "minimum_keypoints_threshold" : -1 + "detection_model": "HUMAN_BODY_ACCURATE", + "body_format": "BODY_38", + "body_selection": "FULL", + "confidence": 40, + "max_range": -1, + "minimum_keypoints_threshold" : -1 }, - "InitFusionParameters": - { - "json_config_filename": "", - "output_performance_metrics": true, - "verbose": false - - }, - "BodyTrackingFusionParameters": - { - "skeleton_minimm_allowed_keypoints": -1, - "skeleton_minimum_allowed_camera": -1, - "skeleton_smoothing": 0 - } + "InitFusionParameters": + { + "json_config_filename": "", + "output_performance_metrics": true, + "verbose": false + }, + "BodyTrackingFusionParameters": + { + "skeleton_minimm_allowed_keypoints": -1, + "skeleton_minimum_allowed_camera": -1, + "skeleton_smoothing": 0 + } } -``` - -To change the name or location of the config file, add it as argument. -For example, to use a file called "ConfigFile.json", run : - -```bash -$ ./ZEDLiveLinkFusion ConfigFile.json -``` -3. You can see the connection status in the terminal. - -- On Windows : - -![](./doc_images/capture_zed_connected.jpg) + ``` +You can get more information about the different parameters role by looking at the [API documentation](https://www.stereolabs.com/docs/api/). -### Setting up a Unreal Engine Project +By default, the Live Link sample will try to open a json file called **ZEDFusionLiveLinkConfig.json** next to the executable. +But you can also give the path to your config file as an argument when running the Live Link sample : +```bash +$ ./ZEDLiveLinkFusion path/to/config/file.json +``` ### Troubleshooting @@ -128,12 +151,11 @@ $ ./ZEDLiveLinkFusion ConfigFile.json - If the ZED Source is not detected in UnrealEngine, enable **Enable by default** in **Edit** -> **Project Settings** -> **UDP Messaging** -> **Enable by default** -![](./doc_images/EnableByDefault.jpg) +![](../images/EnableByDefault.jpg) - You might also need to specify the "Unicast Endpoint" parameter if you are using the same machine for the ZED Livelink sample and the Unreal Project. In this case, set the parameter to "127.0.0.1". - #### On Linux If the plugin crashes at the start, try to run the ldd command onto the sl_zed_c.so library : @@ -146,4 +168,4 @@ It will show all the dependencies required by the .so and allow you to install a Note that the c wrapper used for the Live link plugin is also available here : https://github.com/stereolabs/zed-c-api. -If you encounter issues running the live link plugin, do not hesitate to build the wrapper yourself and place it in the lib/win64 or /linux folder. +If you encounter issues running the live link plugin, do not hesitate to build the wrapper yourself and place it in the `lib/win64` or `lib/linux` folder. diff --git a/zed-livelink-fusion/Release/linux/ZEDFusionLiveLinkConfig.json b/zed-livelink-fusion/Release/linux/ZEDFusionLiveLinkConfig.json index 621e99f..7412845 100644 --- a/zed-livelink-fusion/Release/linux/ZEDFusionLiveLinkConfig.json +++ b/zed-livelink-fusion/Release/linux/ZEDFusionLiveLinkConfig.json @@ -1,7 +1,7 @@ { "InitParameters": { - "resolution": "HD1080", + "resolution": "HD720", "fps": 30, "depth_mode": "ULTRA" }, diff --git a/zed-livelink-fusion/Release/linux/ZEDLiveLinkFusion b/zed-livelink-fusion/Release/linux/ZEDLiveLinkFusion old mode 100644 new mode 100755 index 447ede9..a078fe7 Binary files a/zed-livelink-fusion/Release/linux/ZEDLiveLinkFusion and b/zed-livelink-fusion/Release/linux/ZEDLiveLinkFusion differ diff --git a/zed-livelink-fusion/Release/linux/libsl_zed_c.so b/zed-livelink-fusion/Release/linux/libsl_zed_c.so old mode 100644 new mode 100755 index f2d42d7..a11317c Binary files a/zed-livelink-fusion/Release/linux/libsl_zed_c.so and b/zed-livelink-fusion/Release/linux/libsl_zed_c.so differ diff --git a/zed-livelink-fusion/Release/win64/ZEDLiveLinkFusion.exe b/zed-livelink-fusion/Release/win64/ZEDLiveLinkFusion.exe index 4faec8e..4fb1e4f 100644 Binary files a/zed-livelink-fusion/Release/win64/ZEDLiveLinkFusion.exe and b/zed-livelink-fusion/Release/win64/ZEDLiveLinkFusion.exe differ diff --git a/zed-livelink-fusion/Release/win64/sl_zed_c.dll b/zed-livelink-fusion/Release/win64/sl_zed_c.dll index eab149b..cd28f1d 100644 Binary files a/zed-livelink-fusion/Release/win64/sl_zed_c.dll and b/zed-livelink-fusion/Release/win64/sl_zed_c.dll differ diff --git a/zed-livelink-fusion/ZEDFusionLiveLinkConfig.json b/zed-livelink-fusion/ZEDFusionLiveLinkConfig.json index 621e99f..7412845 100644 --- a/zed-livelink-fusion/ZEDFusionLiveLinkConfig.json +++ b/zed-livelink-fusion/ZEDFusionLiveLinkConfig.json @@ -1,7 +1,7 @@ { "InitParameters": { - "resolution": "HD1080", + "resolution": "HD720", "fps": 30, "depth_mode": "ULTRA" }, diff --git a/zed-livelink-fusion/doc_images/capture_livelink_install.jpg b/zed-livelink-fusion/doc_images/capture_livelink_install.jpg deleted file mode 100644 index 9f777f6..0000000 Binary files a/zed-livelink-fusion/doc_images/capture_livelink_install.jpg and /dev/null differ diff --git a/zed-livelink-fusion/doc_images/capture_livelink_zed.jpg b/zed-livelink-fusion/doc_images/capture_livelink_zed.jpg deleted file mode 100644 index b309693..0000000 Binary files a/zed-livelink-fusion/doc_images/capture_livelink_zed.jpg and /dev/null differ diff --git a/zed-livelink-fusion/doc_images/capture_menu_livelink.jpg b/zed-livelink-fusion/doc_images/capture_menu_livelink.jpg deleted file mode 100644 index 1e81943..0000000 Binary files a/zed-livelink-fusion/doc_images/capture_menu_livelink.jpg and /dev/null differ diff --git a/zed-livelink-fusion/doc_images/capture_plugin.jpg b/zed-livelink-fusion/doc_images/capture_plugin.jpg deleted file mode 100644 index 086fd93..0000000 Binary files a/zed-livelink-fusion/doc_images/capture_plugin.jpg and /dev/null differ diff --git a/zed-livelink-fusion/doc_images/capture_zed_connected.jpg b/zed-livelink-fusion/doc_images/capture_zed_connected.jpg deleted file mode 100644 index e3e6de8..0000000 Binary files a/zed-livelink-fusion/doc_images/capture_zed_connected.jpg and /dev/null differ diff --git a/zed-livelink-fusion/lib/linux/libsl_zed_c.so b/zed-livelink-fusion/lib/linux/libsl_zed_c.so old mode 100644 new mode 100755 index f2d42d7..a11317c Binary files a/zed-livelink-fusion/lib/linux/libsl_zed_c.so and b/zed-livelink-fusion/lib/linux/libsl_zed_c.so differ diff --git a/zed-livelink-fusion/lib/win64/sl_zed_c.dll b/zed-livelink-fusion/lib/win64/sl_zed_c.dll index eab149b..cd28f1d 100644 Binary files a/zed-livelink-fusion/lib/win64/sl_zed_c.dll and b/zed-livelink-fusion/lib/win64/sl_zed_c.dll differ diff --git a/zed-livelink-mono/README.md b/zed-livelink-mono/README.md index 66db016..2095995 100644 --- a/zed-livelink-mono/README.md +++ b/zed-livelink-mono/README.md @@ -15,9 +15,17 @@ The object detection module can be disabled in order to send only camera trackin - For more information, read the ZED [Documentation](https://www.stereolabs.com/docs) and [API documentation](https://www.stereolabs.com/docs/api/) -To compile the tool from source, you will require a source build of Unreal Engine. +To compile the tool from source, you will require a source build of Unreal Engine. For more information, read the [Building ZED Live Link Plugin](https://www.stereolabs.com/docs/livelink/building-the-plugin/) documentation page. -## Download the Engine +## Using ZED Live Link + +We provide you an Unreal project that already contains the different assets necessary to animate avatars properly using the **ZED Live Link** skeleton data. + +You will be able to use directly this project by hitting play while the **ZED Live Link sample** is running and see a default avatar animate. We are showing how to add your own models here: [Animate New Avatar](/livelink/animate-new-avatar/). + +In this tutorial, you'll learn how to use our **ZED Live Link sample**, connect it to our Unreal Project and animate a default avatar with the **Live Link** skeleton data. You must follow these steps: + +### 1. Download the Engine To gain access to the UE engine code, please follow the steps below: @@ -31,72 +39,103 @@ To gain access to the UE engine code, please follow the steps below: > *If the invitation isn't sent, clicking on the following link seems to trigger it: [Unreal Group](https://github.com/orgs/EpicGames). +### 2. Open the Unreal Project -## Build for Windows +Open our *ZEDUnrealLiveLink.uproject* project. -### Compile from Source +* In your project root directory right click on the **ZEDUnrealLiveLink.uproject** file and select **Generate Visual Studio project files** in the context menu. +* Open Visual Studio and compile. Launch the project from the **uproject** file or Visual Studio. -1. Inside the root directory, run **Setup.bat**. This will check all the project dependencies and update them as needed. -2. Clone this repository inside of \Engine\Source\Programs. -3. Run **GenerateProjectFiles.bat** to create project files for the engine. -4. Load the project into Visual Studio by double-clicking on the UE5.sln file, then right click on the ZEDLiveLink target and select **Build**. -5. The binaries will be placed in \Engine\Binaries\Win64\ZEDLiveLink\. +If these options are not available, make sure your Visual Studio install is correctly setup. -### Using the Live Link app + ### 3. Enable the Live Link Plugin in Unreal -The camera input (ip, svo or serial number) can be provided directly as an argument. +You can follow these steps if the Live Link Plugin is not enabled yet in your project. -To send data from a specific camera, add its serial number as arguments. -For example, to connect the camera whose serial number is 10028418, run : +1. Inside your project, In the Menu Bar under **Edit**, select **Plugins**. -```bash -$ ./ZEDLiveLink 10028418 -``` -3. This will open a console window and show the connection status. +![](../images/capture_plugin2.jpg) -## Build for Linux -### Compile from Source +2. Under the Animation section, click Enabled for Live Link, and Yes on the confirmation window, then restart the Editor. -1. Go into the folder you just downloaded/cloned and run **Setup.sh** from the terminal. This will check all the project dependencies and update them as needed. -2. Clone this repository inside of /Engine/Source/Programs. -3. In the Engine root directory, run **GenerateProjectFiles.sh**. -4. Run **make** to build the Engine. -5. Go to UnrealEngine/Engine/Build/BatchFiles and open a terminal. -6. Build the plugin with the command : - ```bash - $ ./RunUAT.sh BuildGraph -Script=Engine/Source/Programs/zed-livelink-plugin/BuildZEDLinux.xml -Target="Stage ZEDLiveLink Linux" - ``` -7. The binaries will be placed in /Engine/Binaries/Linux/ZEDLiveLink/. +![](../images/capture_livelink_install2.jpg) + +### 4. Select the remap asset + +The ZED SDK now has multiple skeleton formats (Body 34, 38, and 70) available for animating a 3D model. +You need to make sure the correct remap asset is selected in the Anim Blueprint of Actor you are using in your level. Indeed, each body format has its own remap asset (as the name and numbers of joint is different). +The remap asset **must** be set in the **Anim blueprint**, in the **ZED Livelink** pose component. -### Using the Live Link app +For example, if you are using the Body format *Body_38*, open the **ABP_ZED_Manny** anim blueprint, select the **ZED LivelinkPose** component and, in the Detail panel, set the **Remap Asset** field to **RemapAssetBody38** -The ZED Live link sample now requires a Config file (Json format) to run. This file contains all the parameters from the ZED SDK that can be modified in this sample. -For more informations about these parameters, the documentation is available here : https://www.stereolabs.com/docs/api/ +![](../images/remap_asset_selection.jpg) -By default, the sample will try to open a json called "ZEDLiveLinkConfig.json" located next to this executable. +### 4. Run ZED Live Link -To change the name or location of the config file, add it as argument. -For example, to use a file called "ConfigFile.json", run : +At this point, you must run the ZED Live Link executable. It will call the ZED SDK in order to compute skeleton data and stream it to the Unreal project. So you must make sure to have the ZED SDK installed on the machine that will run the ZED Live Link sample. + +A pre-built `ZEDLiveLink` sender for the current SDK version is located in `zed-livelink/zed-livelink-mono/Releases/your system/`. + +To learn how to build the ZED Live link sample yourself, take a look at our [Build Zed Live Link](/livelink/building-the-plugin/) documentation page. + +- Open a terminal in the folder containing the ZED Live Link executable, and run ```bash -$ ./ZEDLiveLink ConfigFile.json +$ ./ZEDLiveLink ``` +#### Adjust ZED SDK Parameters + +The ZED SDK parameters are now available in a Json file next to the executable of the Live link sample. +It allows you to change the parameters without re-building the sample. + +The configuration file looks like this : + + ```json +{ + "InitParameters": + { + "input": "USB_ID", + "input_path": 0, + "resolution": "HD1080", + "fps": 30, + "depth_mode": "ULTRA" + }, + "PositionalTrackingParameters": + { + "enable_pose_smoothing" : true, + "enable_area_memory": false + }, + "BodyTrackingParameters": + { + "enable_module": true, + "detection_model": "HUMAN_BODY_ACCURATE", + "body_format": "BODY_38", + "body_selection": "FULL", + "confidence": 40, + "max_range": -1, + "minimum_keypoints_threshold" : -1 + } +} -3. You can see the connection status in the terminal. + ``` -- On Windows : +You can get more information about the different parameters role by looking at the [API documentation](https://www.stereolabs.com/docs/api/). -![](./doc_images/capture_zed_connected.jpg) +By default, the Live Link sample will try to open a json file called **ZEDLiveLinkConfig.json** next to the executable. -- On Linux : +But you can also give the path to your config file as an argument when running the Live Link sample : -![](./doc_images/zed_capture_installed_linux.jpg) +```bash +$ ./ZEDLiveLink path/to/config/file.json +``` + +3. You can see the connection status in the terminal. -### Setting up a Unreal Engine Project +![](../images/capture_zed_connected2.jpg) @@ -106,7 +145,7 @@ Your firewall might block the data stream. If you do not see the ZED Source in t If the ZED Source is not yet detected in UnrealEngine, enable **Enable by default** in **Edit** -> **Project Settings** -> **UDP Messaging** -> **Enable by default** -![](./doc_images/EnableByDefault.jpg) +![](./images/EnableByDefault.jpg) #### On Linux @@ -121,4 +160,4 @@ It will show all the dependencies required by the .so and allow you to install a Note that the c wrapper used for the Live link plugin is also available here : https://github.com/stereolabs/zed-c-api. -If you encounter issues running the live link plugin, do not hesitate to build the wrapper yourself and place it in the lib/win64 or /linux folder. +If you encounter issues running the live link plugin, do not hesitate to build the wrapper yourself and place it in the `lib/win64` or `lib/linux` folder. diff --git a/zed-livelink-mono/Releases/linux/ZEDLiveLink b/zed-livelink-mono/Releases/linux/ZEDLiveLink old mode 100644 new mode 100755 index 6f8561d..ab1b6fd Binary files a/zed-livelink-mono/Releases/linux/ZEDLiveLink and b/zed-livelink-mono/Releases/linux/ZEDLiveLink differ diff --git a/zed-livelink-mono/Releases/linux/libsl_zed_c.so b/zed-livelink-mono/Releases/linux/libsl_zed_c.so old mode 100644 new mode 100755 index f2d42d7..a11317c Binary files a/zed-livelink-mono/Releases/linux/libsl_zed_c.so and b/zed-livelink-mono/Releases/linux/libsl_zed_c.so differ diff --git a/zed-livelink-mono/Releases/win64/ZEDLiveLink.exe b/zed-livelink-mono/Releases/win64/ZEDLiveLink.exe index e6ee425..66e36b2 100644 Binary files a/zed-livelink-mono/Releases/win64/ZEDLiveLink.exe and b/zed-livelink-mono/Releases/win64/ZEDLiveLink.exe differ diff --git a/zed-livelink-mono/Releases/win64/sl_zed_c.dll b/zed-livelink-mono/Releases/win64/sl_zed_c.dll index 407a664..cd28f1d 100644 Binary files a/zed-livelink-mono/Releases/win64/sl_zed_c.dll and b/zed-livelink-mono/Releases/win64/sl_zed_c.dll differ diff --git a/zed-livelink-mono/Source/Private/main.cpp b/zed-livelink-mono/Source/Private/main.cpp index a4cf867..37da7af 100644 --- a/zed-livelink-mono/Source/Private/main.cpp +++ b/zed-livelink-mono/Source/Private/main.cpp @@ -80,7 +80,7 @@ TMap StreamedSkeletons; TArray targetBone; TArray parentsIdx; -/////////////////////////////////////// +//////////////////////////////////////// ///////////// MAIN //////////////////// /////////////////////////////////////// diff --git a/zed-livelink-mono/doc_images/EnableByDefault.jpg b/zed-livelink-mono/doc_images/EnableByDefault.jpg deleted file mode 100644 index 52276e9..0000000 Binary files a/zed-livelink-mono/doc_images/EnableByDefault.jpg and /dev/null differ diff --git a/zed-livelink-mono/doc_images/capture_livelink_camera.jpg b/zed-livelink-mono/doc_images/capture_livelink_camera.jpg deleted file mode 100644 index 62fbc6e..0000000 Binary files a/zed-livelink-mono/doc_images/capture_livelink_camera.jpg and /dev/null differ diff --git a/zed-livelink-mono/doc_images/capture_livelink_install.jpg b/zed-livelink-mono/doc_images/capture_livelink_install.jpg deleted file mode 100644 index 9f777f6..0000000 Binary files a/zed-livelink-mono/doc_images/capture_livelink_install.jpg and /dev/null differ diff --git a/zed-livelink-mono/doc_images/capture_livelink_zed.jpg b/zed-livelink-mono/doc_images/capture_livelink_zed.jpg deleted file mode 100644 index b309693..0000000 Binary files a/zed-livelink-mono/doc_images/capture_livelink_zed.jpg and /dev/null differ diff --git a/zed-livelink-mono/doc_images/capture_menu_livelink.jpg b/zed-livelink-mono/doc_images/capture_menu_livelink.jpg deleted file mode 100644 index 1e81943..0000000 Binary files a/zed-livelink-mono/doc_images/capture_menu_livelink.jpg and /dev/null differ diff --git a/zed-livelink-mono/doc_images/capture_plugin.jpg b/zed-livelink-mono/doc_images/capture_plugin.jpg deleted file mode 100644 index 086fd93..0000000 Binary files a/zed-livelink-mono/doc_images/capture_plugin.jpg and /dev/null differ diff --git a/zed-livelink-mono/doc_images/capture_zed_addsubject.jpg b/zed-livelink-mono/doc_images/capture_zed_addsubject.jpg deleted file mode 100644 index 7c414da..0000000 Binary files a/zed-livelink-mono/doc_images/capture_zed_addsubject.jpg and /dev/null differ diff --git a/zed-livelink-mono/doc_images/capture_zed_connected.jpg b/zed-livelink-mono/doc_images/capture_zed_connected.jpg deleted file mode 100644 index 7a44839..0000000 Binary files a/zed-livelink-mono/doc_images/capture_zed_connected.jpg and /dev/null differ diff --git a/zed-livelink-mono/doc_images/capture_zed_subject_linux.jpg b/zed-livelink-mono/doc_images/capture_zed_subject_linux.jpg deleted file mode 100644 index a42fc8b..0000000 Binary files a/zed-livelink-mono/doc_images/capture_zed_subject_linux.jpg and /dev/null differ diff --git a/zed-livelink-mono/doc_images/livelink_component.jpg b/zed-livelink-mono/doc_images/livelink_component.jpg deleted file mode 100644 index 4afaad6..0000000 Binary files a/zed-livelink-mono/doc_images/livelink_component.jpg and /dev/null differ diff --git a/zed-livelink-mono/doc_images/livelink_component_subject.jpg b/zed-livelink-mono/doc_images/livelink_component_subject.jpg deleted file mode 100644 index ad1a80a..0000000 Binary files a/zed-livelink-mono/doc_images/livelink_component_subject.jpg and /dev/null differ diff --git a/zed-livelink-mono/doc_images/zed_capture_installed_linux.jpg b/zed-livelink-mono/doc_images/zed_capture_installed_linux.jpg deleted file mode 100644 index e05474e..0000000 Binary files a/zed-livelink-mono/doc_images/zed_capture_installed_linux.jpg and /dev/null differ diff --git a/zed-livelink-mono/lib/linux/libsl_zed_c.so b/zed-livelink-mono/lib/linux/libsl_zed_c.so old mode 100644 new mode 100755 index f2d42d7..a11317c Binary files a/zed-livelink-mono/lib/linux/libsl_zed_c.so and b/zed-livelink-mono/lib/linux/libsl_zed_c.so differ diff --git a/zed-livelink-mono/lib/win64/sl_zed_c.dll b/zed-livelink-mono/lib/win64/sl_zed_c.dll index 407a664..cd28f1d 100644 Binary files a/zed-livelink-mono/lib/win64/sl_zed_c.dll and b/zed-livelink-mono/lib/win64/sl_zed_c.dll differ