Various features and fixes

AssetDumper/Source/AssetDumper/Private/AssetDumperModule.cpp

@@ -1,5 +1,6 @@
 #include "AssetDumperModule.h"
 #include "Toolkit/AssetDumping/AssetDumperCommands.h"
+#include "Toolkit/NativeClassDumping/NativeClassDumperCommands.h"
 
 #if METHOD_PATCHING_SUPPORTED
 #include "Patching/NativeHookManager.h"
@@ -19,6 +20,11 @@ void FAssetDumperModule::StartupModule() {
  UE_LOG(LogAssetDumper, Log, TEXT("Game asset dump required through the command line. Game will dump the assets and exit"));
  FAssetDumperCommands::DumpAllGameAssets(FCommandLine::Get());
  }
+
+ if (FParse::Param(FCommandLine::Get(), TEXT("DumpNativeClasses"))) {
+ UE_LOG(LogAssetDumper, Log, TEXT("Native class dump required through the command line. Game will dump the classes and exit"));
+ FNativeClassDumperCommands::DumpAllNativeClasses(FCommandLine::Get());
+ }
 }
 
 void FAssetDumperModule::ShutdownModule() {

AssetDumper/Source/AssetDumper/Private/Toolkit/AssetTypes/AnimationMontageAssetSerializer.cpp

@@ -9,7 +9,7 @@ void UAnimationMontageAssetSerializer::SerializeAsset(TSharedRef<FSerializationC
  BEGIN_ASSET_SERIALIZATION(UAnimMontage)
 
  DISABLE_SERIALIZATION(UAnimSequenceBase, RawCurveData)
- check(Asset->RawCurveData.FloatCurves.Num() == 0);
+ // check(Asset->RawCurveData.FloatCurves.Num() == 0);
 
  SERIALIZE_ASSET_OBJECT
  END_ASSET_SERIALIZATION
@@ -18,3 +18,4 @@ void UAnimationMontageAssetSerializer::SerializeAsset(TSharedRef<FSerializationC
 FName UAnimationMontageAssetSerializer::GetAssetClass() const {
  return UAnimMontage::StaticClass()->GetFName();
 }
+

AssetDumper/Source/AssetDumper/Private/Toolkit/NativeClassDumping/NativeClassDumpProcessor.cpp

@@ -0,0 +1,184 @@
+#include "Toolkit/NativeClassDumping/NativeClassDumpProcessor.h"
+#include "Async/ParallelFor.h"
+#include "Toolkit/ObjectHierarchySerializer.h"
+#include "Toolkit/PropertySerializer.h"
+#include "Toolkit/AssetTypes/AssetHelper.h"
+
+DEFINE_LOG_CATEGORY(LogNativeClassDumper)
+
+FNativeClassDumpSettings::FNativeClassDumpSettings() :
+ RootDumpDirectory(FPaths::ProjectDir() + TEXT("NativeClassDump/")),
+ MaxClassesToProcessInOneTick(FPlatformMisc::NumberOfCores() / 2),
+ MaxStructsToProcessInOneTick(FPlatformMisc::NumberOfCores() / 2),
+ bForceSingleThread(false),
+ bOverwriteExistingFiles(true),
+ bExitOnFinish(false) {
+}
+
+TSharedPtr<FNativeClassDumpProcessor> FNativeClassDumpProcessor::ActiveDumpProcessor = NULL;
+
+FNativeClassDumpProcessor::FNativeClassDumpProcessor(const FNativeClassDumpSettings& Settings, const TArray<UClass*>& InClasses, const TArray<UStruct*>& InStructs) {
+ this->Settings = Settings;
+ this->Classes = InClasses;
+ this->Structs = InStructs;
+ InitializeNativeClassDump();
+}
+
+FNativeClassDumpProcessor::~FNativeClassDumpProcessor() { 
+ this->RemainingClasses.Empty();
+}
+
+TSharedRef<FNativeClassDumpProcessor> FNativeClassDumpProcessor::StartNativeClassDump(const FNativeClassDumpSettings& Settings, const TArray<UClass*>& InClasses, const TArray<UStruct*>& InStructs) {
+ checkf(!ActiveDumpProcessor.IsValid(), TEXT("StartNativeClassDump is called while another native class dump is in progress"));
+
+ TSharedRef<FNativeClassDumpProcessor> NewProcessor = MakeShareable(new FNativeClassDumpProcessor(Settings, InClasses, InStructs));
+ ActiveDumpProcessor = NewProcessor;
+ return NewProcessor;
+}
+
+void FNativeClassDumpProcessor::Tick(float DeltaTime) { 
+ //Process pending dump requests in parallel for loop
+ TArray<UClass*, TInlineAllocator<16>> ClassesToProcessThisTick;
+ ClassesToProcessThisTick.Reserve(Settings.MaxClassesToProcessInOneTick);
+
+ const int32 ClassElementsToCopy = FMath::Min(RemainingClasses.Num(), Settings.MaxClassesToProcessInOneTick);
+ ClassesToProcessThisTick.Append(RemainingClasses.GetData(), ClassElementsToCopy);
+ RemainingClasses.RemoveAt(0, ClassElementsToCopy, false);
+
+ TArray<UStruct*, TInlineAllocator<16>> StructsToProcessThisTick;
+ StructsToProcessThisTick.Reserve(Settings.MaxStructsToProcessInOneTick);
+
+ const int32 StructElementsToCopy = FMath::Min(RemainingStructs.Num(), Settings.MaxStructsToProcessInOneTick);
+ StructsToProcessThisTick.Append(RemainingStructs.GetData(), StructElementsToCopy);
+ RemainingStructs.RemoveAt(0, StructElementsToCopy, false);
+
+ if (ClassesToProcessThisTick.Num()) {
+ EParallelForFlags ParallelFlags = EParallelForFlags::Unbalanced;
+ if (Settings.bForceSingleThread) {
+ ParallelFlags |= EParallelForFlags::ForceSingleThread;
+ }
+ ParallelFor(ClassesToProcessThisTick.Num(), [this, &ClassesToProcessThisTick](const int32 PackageIndex) {
+ PerformNativeClassDumpForClass(ClassesToProcessThisTick[PackageIndex]);
+ }, ParallelFlags);
+ }
+
+ if (StructsToProcessThisTick.Num()) {
+ EParallelForFlags ParallelFlags = EParallelForFlags::Unbalanced;
+ if (Settings.bForceSingleThread) {
+ ParallelFlags |= EParallelForFlags::ForceSingleThread;
+ }
+ ParallelFor(StructsToProcessThisTick.Num(), [this, &StructsToProcessThisTick](const int32 PackageIndex) {
+ PerformNativeClassDumpForStruct(StructsToProcessThisTick[PackageIndex]);
+ }, ParallelFlags);
+ }
+
+ if (RemainingClasses.Num() == 0 && RemainingStructs.Num() == 0) {
+ UE_LOG(LogNativeClassDumper, Display, TEXT("Native class dumping finished successfully"));
+ this->bHasFinishedDumping = true;
+
+ //If we were requested to exit on finish, do it now
+ if (Settings.bExitOnFinish) {
+ UE_LOG(LogNativeClassDumper, Display, TEXT("Exiting because bExitOnFinish was set to true in native class dumper settings..."));
+ FPlatformMisc::RequestExit(false);
+ }
+
+ //If we represent an active dump processor, make sure to reset ourselves once dumping is done
+ if (ActiveDumpProcessor.Get() == this) {
+ ActiveDumpProcessor.Reset();
+ }
+ }
+}
+
+void FNativeClassDumpProcessor::PerformNativeClassDumpForClass(UClass* Class) {
+ const FString Filename = FPackageName::GetShortName(Class->GetPackage()->GetFullName()) / Class->GetName() + TEXT(".json");
+ const FString OutputFilename = Settings.RootDumpDirectory / TEXT("Classes") / Filename;
+
+ if (!Settings.bOverwriteExistingFiles) { 
+ //Skip dumping when we have a dump file already and are not allowed to overwrite files
+ if (FPlatformFileManager::Get().GetPlatformFile().FileExists(*OutputFilename)) {
+ UE_LOG(LogNativeClassDumper, Display, TEXT("Skipping dumping native class %s, dump file is already present and overwriting is not allowed"), *Class->GetFullName());
+ return;
+ }
+ }
+
+ UE_LOG(LogNativeClassDumper, Display, TEXT("Serializing class %s"), *Class->GetFullName());
+
+ const TSharedPtr<FJsonObject> ClassSerializedData = MakeShareable(new FJsonObject());
+ UPropertySerializer* PropertySerializer = NewObject<UPropertySerializer>();
+ PropertySerializer->SetSerializeTransient(true);
+ UObjectHierarchySerializer* ObjectHierarchySerializer = NewObject<UObjectHierarchySerializer>();
+ ObjectHierarchySerializer->SetPropertySerializer(PropertySerializer);
+ ObjectHierarchySerializer->InitializeForSerialization(Class->GetPackage());
+
+ FAssetHelper::SerializeClass(ClassSerializedData, Class, ObjectHierarchySerializer);
+
+ const TSharedRef<FJsonObject> RootObject = MakeShareable(new FJsonObject()); 
+ RootObject->SetObjectField(TEXT("ClassSerializedData"), ClassSerializedData);
+ RootObject->SetArrayField(TEXT("ObjectHierarchy"), ObjectHierarchySerializer->FinalizeSerialization());
+
+ FString ResultString;
+ const TSharedRef<TJsonWriter<>> Writer = TJsonWriterFactory<>::Create(&ResultString);
+ FJsonSerializer::Serialize(RootObject, Writer);
+
+ check(FFileHelper::SaveStringToFile(ResultString, *OutputFilename));
+
+ this->ClassesProcessed.Increment();
+}
+
+void FNativeClassDumpProcessor::PerformNativeClassDumpForStruct(UStruct* Struct) {
+ const FString Filename = FPackageName::GetShortName(Struct->GetPackage()->GetFullName()) / Struct->GetName() + TEXT(".json");
+ const FString OutputFilename = Settings.RootDumpDirectory / TEXT("Structs") / Filename;
+
+ if (!Settings.bOverwriteExistingFiles) { 
+ //Skip dumping when we have a dump file already and are not allowed to overwrite files
+ if (FPlatformFileManager::Get().GetPlatformFile().FileExists(*OutputFilename)) {
+ UE_LOG(LogNativeClassDumper, Display, TEXT("Skipping dumping native class %s, dump file is already present and overwriting is not allowed"), *Struct->GetFullName());
+ return;
+ }
+ }
+
+ UE_LOG(LogNativeClassDumper, Display, TEXT("Serializing struct %s"), *Struct->GetFullName());
+
+ const TSharedPtr<FJsonObject> ClassSerializedData = MakeShareable(new FJsonObject());
+ UPropertySerializer* PropertySerializer = NewObject<UPropertySerializer>();
+ PropertySerializer->SetSerializeTransient(true);
+ UObjectHierarchySerializer* ObjectHierarchySerializer = NewObject<UObjectHierarchySerializer>();
+ ObjectHierarchySerializer->SetPropertySerializer(PropertySerializer);
+ ObjectHierarchySerializer->InitializeForSerialization(Struct->GetPackage());
+
+ FAssetHelper::SerializeStruct(ClassSerializedData, Struct, ObjectHierarchySerializer);
+
+ const TSharedRef<FJsonObject> RootObject = MakeShareable(new FJsonObject()); 
+ RootObject->SetObjectField(TEXT("ClassSerializedData"), ClassSerializedData);
+ RootObject->SetArrayField(TEXT("ObjectHierarchy"), ObjectHierarchySerializer->FinalizeSerialization());
+
+ FString ResultString;
+ const TSharedRef<TJsonWriter<>> Writer = TJsonWriterFactory<>::Create(&ResultString);
+ FJsonSerializer::Serialize(RootObject, Writer);
+
+ check(FFileHelper::SaveStringToFile(ResultString, *OutputFilename));
+
+ this->ClassesProcessed.Increment();
+}
+
+bool FNativeClassDumpProcessor::IsTickable() const {
+ return bHasFinishedDumping == false;
+}
+
+TStatId FNativeClassDumpProcessor::GetStatId() const {
+ RETURN_QUICK_DECLARE_CYCLE_STAT(FNativeClassDumpProcessor, STATGROUP_Game);
+}
+
+bool FNativeClassDumpProcessor::IsTickableWhenPaused() const {
+ return true;
+}
+
+void FNativeClassDumpProcessor::InitializeNativeClassDump() {
+ this->Settings = Settings;
+ this->bHasFinishedDumping = false;
+ this->RemainingClasses = Classes;
+ this->RemainingStructs = Structs;
+ this->ClassesTotal = Classes.Num();
+ check(ClassesTotal);
+ UE_LOG(LogNativeClassDumper, Display, TEXT("Starting native class dump of %d classes..."), ClassesTotal);
+}

AssetDumper/Source/AssetDumper/Private/Toolkit/NativeClassDumping/NativeClassDumperCommands.cpp

@@ -0,0 +1,51 @@
+#include "Toolkit/NativeClassDumping/NativeClassDumperCommands.h"
+#include "Toolkit/NativeClassDumping/NativeClassDumpProcessor.h"
+
+#define LOCTEXT_NAMESPACE "NativeClassDumper"
+
+void FNativeClassDumperCommands::DumpAllNativeClasses(const FString& Params) {
+
+ TArray<UClass*> Classes;
+
+ for (TObjectIterator<UClass> ClassIt; ClassIt; ++ClassIt) {
+ UClass* Class = *ClassIt;
+
+ // Only interested in native C++ classes
+ if (!Class->IsNative()) {
+ continue;
+ }
+
+ Classes.Add(Class);
+ }
+
+ TArray<UStruct*> Structs;
+
+ for (TObjectIterator<UStruct> StructIt; StructIt; ++StructIt) {
+ UStruct* Struct = *StructIt;
+
+ // Only interested in native C++ *structs*
+ if (!Struct->IsNative() || Struct->IsA<UClass>() || Struct->IsA<UFunction>()) {
+ continue;
+ }
+
+ Structs.Add(Struct);
+ }
+
+ UE_LOG(LogNativeClassDumper, Log, TEXT("Asset data gathered successfully! Gathered %d native classes and %d native structs for dumping"), Classes.Num(), Structs.Num());
+
+ FNativeClassDumpSettings DumpSettings{};
+ DumpSettings.bExitOnFinish = true;
+ FNativeClassDumpProcessor::StartNativeClassDump(DumpSettings, Classes, Structs);
+
+ UE_LOG(LogNativeClassDumper, Log, TEXT("Native class dump started successfully, game will shutdown on finish"));
+}
+
+void DumpAllNativeClasses(const TArray<FString>& Args, UWorld* World, FOutputDevice& Ar)
+{
+ Ar.Log(TEXT("Starting console-driven native class dumping, dumping all native classes"));
+ FNativeClassDumperCommands::DumpAllNativeClasses(FString::Join(Args, TEXT(" ")));
+}
+static FAutoConsoleCommand DumpAllNativeClassesCommand(
+ TEXT("DumpAllNativeClasses"),
+ TEXT("Dumps all native classes existing in the game"),
+ FConsoleCommandWithWorldArgsAndOutputDeviceDelegate::CreateStatic(&DumpAllNativeClasses));

AssetDumper/Source/AssetDumper/Private/Toolkit/ObjectHierarchySerializer.cpp

@@ -223,13 +223,12 @@ bool UObjectHierarchySerializer::CompareObjectsWithContext(const int32 ObjectInd
 
  if (ObjectIndex == INDEX_NONE)
  return false;
-
+ 
  // If the object is not found, deserializing it would still be NULL
  const TSharedPtr<FJsonObject>& ObjectJson = SerializedObjects.FindChecked(ObjectIndex);
  const FString ObjectType = ObjectJson->GetStringField(TEXT("Type"));
 
  return ObjectType == TEXT("Import") && DeserializeObject(ObjectIndex) == NULL;
- //return ObjectIndex == INDEX_NONE && Object == NULL;
  }
 
  //Return true if we have already compared this object, otherwise we will run into the recursion
@@ -523,6 +522,11 @@ UObject* UObjectHierarchySerializer::DeserializeExportedObject(int32 ObjectIndex
  ConstructedObject = ExistingObject;
 
  } else {
+ UObject* ExistingObjectWithDifferentClass = StaticFindObjectFast(UObject::StaticClass(), OuterObject, *ObjectName);
+ if (ExistingObjectWithDifferentClass != nullptr) {
+ UE_LOG(LogObjectHierarchySerializer, Warning, TEXT("Object %s already exists with different class %s. Moving object and recreating."), *ObjectName, *ExistingObjectWithDifferentClass->GetClass()->GetName());
+ ExistingObjectWithDifferentClass->Rename(nullptr, GetTransientPackage(), REN_DontCreateRedirectors);
+ }
  //Construct new object if we cannot otherwise
  const EObjectFlags ObjectLoadFlags = (EObjectFlags) ObjectJson->GetIntegerField(TEXT("ObjectFlags"));
 

AssetDumper/Source/AssetDumper/Private/Toolkit/PropertySerializer.cpp

@@ -96,6 +96,8 @@ UPropertySerializer::UPropertySerializer() {
 
  this->StructSerializers.Add(DateTimeStruct, MakeShared<FDateTimeSerializer>());
  this->StructSerializers.Add(TimespanStruct, MakeShared<FTimespanSerializer>());
+
+ this->SerializeTransient = false;
 }
 
 void UPropertySerializer::DeserializePropertyValue(FProperty* Property, const TSharedRef<FJsonValue>& JsonValue, void* Value) {
@@ -297,7 +299,7 @@ void UPropertySerializer::AddStructSerializer(UScriptStruct* Struct, const TShar
 
 bool UPropertySerializer::ShouldSerializeProperty(FProperty* Property) const {
  //skip transient properties
- if (Property->HasAnyPropertyFlags(CPF_Transient)) {
+ if (Property->HasAnyPropertyFlags(CPF_Transient) && !SerializeTransient) {
  return false;
  }
  //Skip editor only properties altogether
@@ -712,7 +714,7 @@ bool UPropertySerializer::ComparePropertyValuesInner(FProperty* Property, const
 
  if (const FTextProperty* TextProperty = CastField<const FTextProperty>(Property)) {
  // FTextProperty::Identical compares the CultureInvariant flag, and sometimes empty deserialized texts don't have it while the exiting texts do
- if (TextProperty->GetPropertyValue(CurrentValue).IsEmpty() && TextProperty->GetPropertyValue(DeserializedElement.GetObjAddress()).IsEmpty())
+ if(TextProperty->GetPropertyValue(CurrentValue).IsEmpty() && TextProperty->GetPropertyValue(DeserializedElement.GetObjAddress()).IsEmpty())
  return true;
  }
 

AssetDumper/Source/AssetDumper/Public/Toolkit/NativeClassDumping/NativeClassDumpProcessor.h

@@ -0,0 +1,68 @@
+#pragma once
+#include "CoreMinimal.h"
+#include "Tickable.h"
+
+DECLARE_LOG_CATEGORY_EXTERN(LogNativeClassDumper, All, All);
+
+/** Holds asset dumping related settings */
+struct ASSETDUMPER_API FNativeClassDumpSettings {
+ FString RootDumpDirectory;
+ int32 MaxClassesToProcessInOneTick;
+ int32 MaxStructsToProcessInOneTick;
+ bool bForceSingleThread;
+ bool bOverwriteExistingFiles;
+ bool bExitOnFinish;
+
+ /** Default settings for native class dumping */
+ FNativeClassDumpSettings();
+};
+
+/**
+ * This class is responsible for processing native class dumping request
+ * Only one instance of this class can be active at a time
+ * Global active instance of the native class dump processor can be retrieved through GetActiveInstance() method
+ */
+class ASSETDUMPER_API FNativeClassDumpProcessor : public FTickableGameObject {
+private:
+ static TSharedPtr<FNativeClassDumpProcessor> ActiveDumpProcessor;
+
+ TArray<UClass*> Classes;
+ TArray<UClass*> RemainingClasses;
+ TArray<UStruct*> Structs;
+ TArray<UStruct*> RemainingStructs;
+
+ int32 ClassesTotal;
+ FThreadSafeCounter ClassesSkipped;
+ FThreadSafeCounter ClassesProcessed;
+ FNativeClassDumpSettings Settings;
+ bool bHasFinishedDumping;
+
+ explicit FNativeClassDumpProcessor(const FNativeClassDumpSettings& Settings, const TArray<UClass*>& InClasses, const TArray<UStruct*>& InStructs);
+public:
+ ~FNativeClassDumpProcessor();
+
+ /** Returns currently active instance of the dump processor. Try not to hold any long-living references to it, as it will prevent it's garbage collection */
+ FORCEINLINE static TSharedPtr<FNativeClassDumpProcessor> GetActiveDumpProcessor() { return ActiveDumpProcessor; }
+
+ /** Begins native class dumping with specified settings for provided native class. Crashes when dumping is already in progress */
+ static TSharedRef<FNativeClassDumpProcessor> StartNativeClassDump(const FNativeClassDumpSettings& Settings, const TArray<UClass*>& InClasses, const TArray<UStruct*>& InStructs);
+
+ /** Returns current progress of the native class dumping */
+ FORCEINLINE float GetProgressPct() const { return (ClassesSkipped.GetValue() + ClassesProcessed.GetValue()) / (ClassesTotal * 1.0f); }
+
+ FORCEINLINE int32 GetTotalClasses() const { return ClassesTotal; }
+ FORCEINLINE int32 GetClassesSkipped() const { return ClassesSkipped.GetValue(); }
+ FORCEINLINE int32 GetClassesProcessed() const { return ClassesProcessed.GetValue(); }
+ FORCEINLINE bool IsFinishedDumping() const { return bHasFinishedDumping; }
+
+ //Begin FTickableGameObject
+ virtual void Tick(float DeltaTime) override;
+ virtual bool IsTickable() const override;
+ virtual TStatId GetStatId() const override;
+ virtual bool IsTickableWhenPaused() const override;
+ //End FTickableGameObject
+protected:
+ void InitializeNativeClassDump();
+ void PerformNativeClassDumpForClass(UClass* Class);
+ void PerformNativeClassDumpForStruct(UStruct* Struct);
+};