Skip to content

Commit

Permalink
Rework chunk creatin for model, animrig, animseqs
Browse files Browse the repository at this point in the history 
For models, the name and guid refs are always stored in 1 chunk, so we only ever create 1 CPU chunk for it now.
For animrigs, the name, rmdl data and guid refs are always stored in 1 chunk, but only the guid refs are aligned to 8 bytes. We now also only ever create 1 CPU chunk for this.
For animseqs, name and rmdl data reside in a single CPU chunk, that is aligned to 1 byte. Previously this was 64, but this is incorrect.

The resulting pak file is slightly smaller, while also more stable in-game now.
  • Loading branch information
@Mauler125
Mauler125 committed Dec 17, 2024
1 parent d776575 commit 90bbc73
Show file tree
Hide file tree
Showing 3 changed files with 136 additions and 99 deletions.
49 changes: 33 additions & 16 deletions src/assets/animrig.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,51 +4,68 @@
#include "public/material.h"
#include <public/animrig.h>

extern bool AnimSeq_AddSequenceRefs(CPakFile* const pak, CPakDataChunk* const chunk, uint32_t* const sequenceCount, const rapidjson::Value& mapEntry);
extern PakGuid_t* AnimSeq_AutoAddSequenceRefs(CPakFile* const pak, uint32_t* const sequenceCount, const rapidjson::Value& mapEntry);

// anim rigs are stored in rmdl's. use this to read it out.
extern char* Model_ReadRMDLFile(const std::string& path, const uint64_t alignment);

// page chunk structure and order:
// - header HEAD (align=8)
// - data CPU (align=8) name, rmdl then refs. name and rmdl are aligned to 1 byte, refs are 8 (padded from rmdl buffer)
void Assets::AddAnimRigAsset_v4(CPakFile* const pak, const PakGuid_t assetGuid, const char* const assetPath, const rapidjson::Value& mapEntry)
{
// deal with dependencies first; auto-add all animation sequences.
CPakDataChunk sequenceRefChunk; uint32_t sequenceCount;
const bool hasAnimSeqRefs = AnimSeq_AddSequenceRefs(pak, &sequenceRefChunk, &sequenceCount, mapEntry);
uint32_t sequenceCount = 0;
PakGuid_t* const sequenceRefs = AnimSeq_AutoAddSequenceRefs(pak, &sequenceCount, mapEntry);

// from here we start with creating chunks for the target animrig asset.
CPakDataChunk hdrChunk = pak->CreateDataChunk(sizeof(AnimRigAssetHeader_t), SF_HEAD, 8);
const size_t assetNameLength = strlen(assetPath);

CPakDataChunk nameChunk = pak->CreateDataChunk(assetNameLength + 1, SF_CPU, 1); // [rika]: only aligned to 1 byte in season 3 paks
memcpy_s(nameChunk.Data(), assetNameLength, assetPath, assetNameLength);
AnimRigAssetHeader_t* const pHdr = reinterpret_cast<AnimRigAssetHeader_t*>(hdrChunk.Data());

// open and validate file to get buffer
char* const animRigFileBuffer = Model_ReadRMDLFile(pak->GetAssetPath() + assetPath, 8);
const studiohdr_t* const studiohdr = reinterpret_cast<const studiohdr_t*>(animRigFileBuffer);

CPakDataChunk rigChunk = pak->CreateDataChunk(studiohdr->length, SF_CPU, 8, animRigFileBuffer);
// note: both of these are aligned to 1 byte, but we pad the rmdl buffer as
// the guid ref block needs to be aligned to 8 bytes.
const size_t assetNameBufLen = strlen(assetPath) + 1;
const size_t rmdlBufLen = IALIGN8(studiohdr->length);

AnimRigAssetHeader_t* const pHdr = reinterpret_cast<AnimRigAssetHeader_t*>(hdrChunk.Data());
pHdr->data = rigChunk.GetPointer();
pHdr->name = nameChunk.GetPointer();
const size_t sequenceRefBufLen = sequenceCount * sizeof(PakGuid_t);

pak->AddPointer(hdrChunk.GetPointer(offsetof(AnimRigAssetHeader_t, data)));
CPakDataChunk rigChunk = pak->CreateDataChunk(assetNameBufLen + rmdlBufLen + sequenceRefBufLen, SF_CPU, 8);
char* const nameBuf = rigChunk.Data();

memcpy(nameBuf, assetPath, assetNameBufLen);
pHdr->name = rigChunk.GetPointer();
pak->AddPointer(hdrChunk.GetPointer(offsetof(AnimRigAssetHeader_t, name)));

studiohdr_t* const studioBuf = reinterpret_cast<studiohdr_t*>(&rigChunk.Data()[assetNameBufLen]);

memcpy(studioBuf, animRigFileBuffer, studiohdr->length);
pHdr->data = rigChunk.GetPointer(assetNameBufLen);
pak->AddPointer(hdrChunk.GetPointer(offsetof(AnimRigAssetHeader_t, data)));

delete[] animRigFileBuffer;
std::vector<PakGuidRefHdr_t> guids;

if (hasAnimSeqRefs)
if (sequenceRefs)
{
guids.resize(sequenceCount);
const size_t base = assetNameBufLen + rmdlBufLen;
PakGuid_t* const sequenceRefBuf = reinterpret_cast<PakGuid_t*>(&rigChunk.Data()[base]);

memcpy(sequenceRefBuf, sequenceRefs, sequenceRefBufLen);

pHdr->sequenceCount = sequenceCount;
pHdr->pSequences = sequenceRefChunk.GetPointer();
pHdr->pSequences = rigChunk.GetPointer(base);

pak->AddPointer(hdrChunk.GetPointer(offsetof(AnimRigAssetHeader_t, pSequences)));

guids.resize(sequenceCount);

for (uint32_t i = 0; i < sequenceCount; ++i)
{
guids[i] = sequenceRefChunk.GetPointer(i * sizeof(PakGuid_t));
guids[i] = rigChunk.GetPointer(base + (i * sizeof(PakGuid_t)));
}
}

Expand Down
39 changes: 19 additions & 20 deletions src/assets/animseq.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,9 @@
#include "assets.h"
#include "public/studio.h"

// page chunk structure and order:
// - header HEAD (align=8)
// - data CPU (align=1) name, then rmdl. unlike models, this is aligned to 1 since we don't have BVH4 collision data here.
static void AnimSeq_InternalAddAnimSeq(CPakFile* const pak, const PakGuid_t assetGuid, const char* const assetPath)
{
const std::string rseqFilePath = pak->GetAssetPath() + assetPath;
Expand All @@ -17,38 +20,37 @@ static void AnimSeq_InternalAddAnimSeq(CPakFile* const pak, const PakGuid_t asse

CPakDataChunk hdrChunk = pak->CreateDataChunk(sizeof(AnimSeqAssetHeader_t), SF_HEAD, 8);

const size_t rseqNameLenAligned = IALIGN4(strlen(assetPath) + 1);
const size_t rseqNameBufLen = strlen(assetPath) + 1;
const size_t rseqFileSize = rseqInput.GetSize();

CPakDataChunk dataChunk = pak->CreateDataChunk(IALIGN4(rseqNameLenAligned + rseqFileSize), SF_CPU, 64);
CPakDataChunk dataChunk = pak->CreateDataChunk(rseqNameBufLen + rseqFileSize, SF_CPU, 1);

// write the rseq file path into the data buffer
snprintf(dataChunk.Data(), rseqNameLenAligned, "%s", assetPath);
memcpy(dataChunk.Data(), assetPath, rseqNameBufLen);

// write the rseq data into the data buffer
rseqInput.Read(dataChunk.Data() + rseqNameLenAligned, rseqFileSize);
rseqInput.Read(dataChunk.Data() + rseqNameBufLen, rseqFileSize);
rseqInput.Close();

const mstudioseqdesc_t& seqdesc = *reinterpret_cast<mstudioseqdesc_t*>(dataChunk.Data() + rseqNameLenAligned);

const mstudioseqdesc_t& seqdesc = *reinterpret_cast<mstudioseqdesc_t*>(dataChunk.Data() + rseqNameBufLen);
AnimSeqAssetHeader_t* const aseqHeader = reinterpret_cast<AnimSeqAssetHeader_t*>(hdrChunk.Data());
aseqHeader->szname = dataChunk.GetPointer();

aseqHeader->data = dataChunk.GetPointer(rseqNameLenAligned);
aseqHeader->szname = dataChunk.GetPointer();
aseqHeader->data = dataChunk.GetPointer(rseqNameBufLen);

pak->AddPointer(hdrChunk.GetPointer(offsetof(AnimSeqAssetHeader_t, szname)));
pak->AddPointer(hdrChunk.GetPointer(offsetof(AnimSeqAssetHeader_t, data)));

std::vector<PakGuidRefHdr_t> guids;

rmem dataBuf(dataChunk.Data());
dataBuf.seek(rseqNameLenAligned + seqdesc.autolayerindex, rseekdir::beg);
dataBuf.seek(rseqNameBufLen + seqdesc.autolayerindex, rseekdir::beg);

// Iterate over each of the sequence's autolayers to register each of the autolayer GUIDs
// This is required as otherwise the game will crash while trying to dereference a non-converted GUID.
for (int i = 0; i < seqdesc.numautolayers; ++i)
{
dataBuf.seek(rseqNameLenAligned + seqdesc.autolayerindex + (i * sizeof(mstudioautolayer_t)), rseekdir::beg);
dataBuf.seek(rseqNameBufLen + seqdesc.autolayerindex + (i * sizeof(mstudioautolayer_t)), rseekdir::beg);

const mstudioautolayer_t* const autolayer = dataBuf.get<const mstudioautolayer_t>();

Expand Down Expand Up @@ -76,24 +78,20 @@ static void AnimSeq_InternalAddAnimSeq(CPakFile* const pak, const PakGuid_t asse
pak->PushAsset(asset);
}

bool AnimSeq_AddSequenceRefs(CPakFile* const pak, CPakDataChunk* const chunk, uint32_t* const sequenceCount, const rapidjson::Value& mapEntry)
PakGuid_t* AnimSeq_AutoAddSequenceRefs(CPakFile* const pak, uint32_t* const sequenceCount, const rapidjson::Value& mapEntry)
{
rapidjson::Value::ConstMemberIterator sequencesIt;

if (!JSON_GetIterator(mapEntry, "$sequences", JSONFieldType_e::kArray, sequencesIt))
return false;
return nullptr;

const rapidjson::Value::ConstArray sequencesArray = sequencesIt->value.GetArray();

if (sequencesArray.Empty())
return false;
return nullptr;

const size_t numSequences = sequencesArray.Size();

(*chunk) = pak->CreateDataChunk(sizeof(PakGuid_t) * numSequences, SF_CPU, 8);
(*sequenceCount) = static_cast<uint32_t>(numSequences);

PakGuid_t* const pGuids = reinterpret_cast<PakGuid_t*>(chunk->Data());
PakGuid_t* const guidBuf = new PakGuid_t[numSequences];

int seqIndex = -1;
for (const auto& sequence : sequencesArray)
Expand Down Expand Up @@ -121,10 +119,11 @@ bool AnimSeq_AddSequenceRefs(CPakFile* const pak, CPakDataChunk* const chunk, ui
AnimSeq_InternalAddAnimSeq(pak, guid, sequencePath);
}

pGuids[seqIndex] = guid;
guidBuf[seqIndex] = guid;
}

return true;
(*sequenceCount) = static_cast<uint32_t>(numSequences);
return guidBuf;
}

void Assets::AddAnimSeqAsset_v7(CPakFile* const pak, const PakGuid_t assetGuid, const char* const assetPath, const rapidjson::Value& mapEntry)
Expand Down
147 changes: 84 additions & 63 deletions src/assets/model.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,24 +60,20 @@ static char* Model_ReadVGFile(const std::string& path, size_t* const pFileSize)
return buf;
}

static bool Model_AddAnimRigRefs(CPakFile* const pak, CPakDataChunk* const chunk, uint32_t* const sequenceCount, const rapidjson::Value& mapEntry)
static PakGuid_t* Model_AddAnimRigRefs(CPakFile* const pak, uint32_t* const sequenceCount, const rapidjson::Value& mapEntry)
{
rapidjson::Value::ConstMemberIterator it;

if (!JSON_GetIterator(mapEntry, "$animrigs", JSONFieldType_e::kArray, it))
return false;
return nullptr;

const rapidjson::Value::ConstArray animrigs = it->value.GetArray();

if (animrigs.Empty())
return false;
return nullptr;

const uint32_t numAnimrigs = (uint32_t)animrigs.Size();

(*chunk) = pak->CreateDataChunk(numAnimrigs * sizeof(PakGuid_t), SF_CPU, 8);
(*sequenceCount) = static_cast<uint32_t>(animrigs.Size());

rmem arigBuf(chunk->Data());
const size_t numAnimrigs = animrigs.Size();
PakGuid_t* const guidBuf = new PakGuid_t[numAnimrigs];

int i = -1;
for (const auto& animrig : animrigs)
Expand All @@ -88,15 +84,80 @@ static bool Model_AddAnimRigRefs(CPakFile* const pak, CPakDataChunk* const chunk
if (!guid)
Error("Unable to parse animrig #%i.\n", i);

arigBuf.write<PakGuid_t>(guid);
guidBuf[i] = guid;

// check if anim rig is a local asset so that the relation can be added
PakAsset_t* const animRigAsset = pak->GetAssetByGuid(guid);

pak->SetCurrentAssetAsDependentForAsset(animRigAsset);
}

return true;
(*sequenceCount) = static_cast<uint32_t>(animrigs.Size());
return guidBuf;
}

static void Model_AllocateIntermediateDataChunk(CPakFile* const pak, CPakDataChunk& hdrChunk, ModelAssetHeader_t* const pHdr,
PakGuid_t* const animrigRefs, const uint32_t animrigCount, PakGuid_t* const sequenceRefs, const uint32_t sequenceCount,
std::vector<PakGuidRefHdr_t>& guids, const char* const assetPath)
{
// the model name is aligned to 1 byte, but the guid ref block is aligned
// to 8, we have to pad the name buffer to align the guid ref block. if
// we have no guid ref blocks, the entire chunk will be aligned to 1 byte.
const size_t modelNameBufLen = strlen(assetPath) + 1;
const size_t alignedNameBufLen = IALIGN8(modelNameBufLen);

const size_t animRigRefsBufLen = animrigCount * sizeof(PakGuid_t);
const size_t sequenceRefsBufLen = sequenceCount * sizeof(PakGuid_t);

const bool hasGuidRefs = animrigRefs || sequenceRefs;

CPakDataChunk intermediateChunk = pak->CreateDataChunk(alignedNameBufLen + animRigRefsBufLen + sequenceRefsBufLen, SF_CPU, hasGuidRefs ? 8 : 1);
memcpy(intermediateChunk.Data(), assetPath, modelNameBufLen); // Write the null-terminated asset path to the chunk buffer.

pHdr->pName = intermediateChunk.GetPointer();
pak->AddPointer(hdrChunk.GetPointer(offsetof(ModelAssetHeader_t, pName)));

if (hasGuidRefs)
{
guids.resize(animrigCount + sequenceCount);
uint64_t curIndex = 0;

if (animrigRefs)
{
const size_t base = alignedNameBufLen;

memcpy(&intermediateChunk.Data()[base], animrigRefs, animRigRefsBufLen);
delete[] animrigRefs;

pHdr->animRigCount = animrigCount;
pHdr->pAnimRigs = intermediateChunk.GetPointer(base);

pak->AddPointer(hdrChunk.GetPointer(offsetof(ModelAssetHeader_t, pAnimRigs)));

for (uint32_t i = 0; i < animrigCount; ++i)
{
guids[curIndex++] = intermediateChunk.GetPointer(base + (sizeof(PakGuid_t) * i));
}
}

if (sequenceRefs)
{
const size_t base = alignedNameBufLen + (curIndex * sizeof(PakGuid_t));

memcpy(&intermediateChunk.Data()[base], sequenceRefs, sequenceRefsBufLen);
delete[] sequenceRefs;

pHdr->sequenceCount = sequenceCount;
pHdr->pSequences = intermediateChunk.GetPointer(base);

pak->AddPointer(hdrChunk.GetPointer(offsetof(ModelAssetHeader_t, pSequences)));

for (uint32_t i = 0; i < sequenceCount; ++i)
{
guids[curIndex++] = intermediateChunk.GetPointer(base + (sizeof(PakGuid_t) * i));
}
}
}
}

static uint64_t Model_InternalAddVertexGroupData(CPakFile* const pak, CPakDataChunk* const hdrChunk, ModelAssetHeader_t* const modelHdr, studiohdr_t* const studiohdr, const std::string& rmdlFilePath)
Expand Down Expand Up @@ -132,74 +193,34 @@ static uint64_t Model_InternalAddVertexGroupData(CPakFile* const pak, CPakDataCh
return de.offset;
}

extern bool AnimSeq_AddSequenceRefs(CPakFile* const pak, CPakDataChunk* const chunk, uint32_t* const sequenceCount, const rapidjson::Value& mapEntry);
extern PakGuid_t* AnimSeq_AutoAddSequenceRefs(CPakFile* const pak, uint32_t* const sequenceCount, const rapidjson::Value& mapEntry);

// chunk creation order:
// page chunk structure and order:
// - header HEAD (align=8)
// - name CPU (align=1)
// - animrig guid CPU (align=8) // typically merged into name chunk
// - intermediate CPU (align=1?8) name, animrig refs then animseqs refs. aligned to 1 if we don't have any refs.
// - vphysics TEMP (align=1)
// - vertex groups TEMP_CLIENT (align=1)
// - rmdl CPU (align=64)
// - rmdl CPU (align=64) 64 bit aligned because collision data is loaded with aligned SIMD instructions.
void Assets::AddModelAsset_v9(CPakFile* const pak, const PakGuid_t assetGuid, const char* const assetPath, const rapidjson::Value& mapEntry)
{
// deal with dependencies first; auto-add all animation sequences and animation rigs.
CPakDataChunk sequenceRefChunk; uint32_t sequenceCount = 0;
const bool hasSequenceRefs = AnimSeq_AddSequenceRefs(pak, &sequenceRefChunk, &sequenceCount, mapEntry);
// deal with dependencies first; auto-add all animation sequences.
uint32_t sequenceCount = 0;
PakGuid_t* const sequenceRefs = AnimSeq_AutoAddSequenceRefs(pak, &sequenceCount, mapEntry);

CPakDataChunk animrigRefChunk; uint32_t animrigCount = 0;
const bool hasAnimrigRefs = Model_AddAnimRigRefs(pak, &animrigRefChunk, &animrigCount, mapEntry);
// this function only creates the arig guid refs, it does not auto-add.
uint32_t animrigCount = 0;
PakGuid_t* const animrigRefs = Model_AddAnimRigRefs(pak, &animrigCount, mapEntry);

// from here we start with creating chunks for the target model asset.
CPakDataChunk hdrChunk = pak->CreateDataChunk(sizeof(ModelAssetHeader_t), SF_HEAD, 8);
ModelAssetHeader_t* const pHdr = reinterpret_cast<ModelAssetHeader_t*>(hdrChunk.Data());

// the name chunk comes directly after the header in bakery files.
const size_t modelNameBufLen = strlen(assetPath) + 1;

CPakDataChunk nameChunk = pak->CreateDataChunk(modelNameBufLen, SF_CPU, 8);
memcpy(nameChunk.Data(), assetPath, modelNameBufLen); // Write the null-terminated asset path to the name chunk.

pHdr->pName = nameChunk.GetPointer();
pak->AddPointer(hdrChunk.GetPointer(offsetof(ModelAssetHeader_t, pName)));

std::vector<PakGuidRefHdr_t> guids;

//
// Animseqs / Anim Rigs
// Name, Anim Rigs and Animseqs, these all share 1 data chunk.
//
if (hasSequenceRefs || hasAnimrigRefs)
{
guids.resize(sequenceCount + animrigCount);

uint64_t curIndex = 0;

if (hasSequenceRefs)
{
pHdr->sequenceCount = sequenceCount;
pHdr->pSequences = sequenceRefChunk.GetPointer();

pak->AddPointer(hdrChunk.GetPointer(offsetof(ModelAssetHeader_t, pSequences)));

for (uint32_t i = 0; i < sequenceCount; ++i)
{
guids[curIndex++] = sequenceRefChunk.GetPointer(sizeof(PakGuid_t) * i);
}
}

if (hasAnimrigRefs)
{
pHdr->animRigCount = animrigCount;
pHdr->pAnimRigs = animrigRefChunk.GetPointer();

pak->AddPointer(hdrChunk.GetPointer(offsetof(ModelAssetHeader_t, pAnimRigs)));

for (uint32_t j = 0; j < pHdr->animRigCount; ++j)
{
guids[curIndex++] = animrigRefChunk.GetPointer(sizeof(PakGuid_t) * j);
}
}
}
Model_AllocateIntermediateDataChunk(pak, hdrChunk, pHdr, animrigRefs, animrigCount, sequenceRefs, sequenceCount, guids, assetPath);

const std::string rmdlFilePath = pak->GetAssetPath() + assetPath;

Expand Down

0 comments on commit 90bbc73

Please sign in to comment.