Skip to content

Commit

Permalink
22068: Reduces memory use of store operations (#301)
Browse files Browse the repository at this point in the history
  • Loading branch information
@howsohazard
howsohazard authored Nov 4, 2024
1 parent 2d96164 commit 5c6a9f1
Show file tree
Hide file tree
Showing 5 changed files with 275 additions and 168 deletions.
61 changes: 54 additions & 7 deletions src/Amalgam/AssetManager.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

//project headers:
#include "AmalgamVersion.h"
#include "BinaryPacking.h"
#include "Entity.h"
#include "EntityExternalInterface.h"
#include "EntityManipulation.h"
Expand Down Expand Up @@ -128,6 +129,58 @@ class AssetManager
Entity *LoadEntityFromResource(AssetParameters &asset_params, bool persistent,
std::string default_random_seed, Interpreter *calling_interpreter, EntityExternalInterface::LoadEntityStatus &status);

//Flattens entity piece-by-piece in a manner to reduce memory when storing
template<typename EntityReferenceType = EntityReadReference>
bool FlattenAndStoreEntityToResource(Entity *entity, AssetParameters &asset_params,
Entity::EntityReferenceBufferReference<EntityReferenceType> &all_contained_entities)
{
EvaluableNode *top_entity_code = EntityManipulation::FlattenOnlyTopEntity(&entity->evaluableNodeManager,
entity, asset_params.includeRandSeeds, true);
std::string code_string = Parser::Unparse(top_entity_code, &entity->evaluableNodeManager,
asset_params.prettyPrint, true, asset_params.sortKeys, true);
entity->evaluableNodeManager.FreeNodeTree(top_entity_code);

//loop over contained entities, freeing resources after each entity
for(size_t i = 0; i < all_contained_entities->size(); i++)
{
auto &cur_entity = (*all_contained_entities)[i];
EvaluableNode *create_entity_code = EntityManipulation::FlattenOnlyOneContainedEntity(
&entity->evaluableNodeManager, cur_entity, entity, asset_params.includeRandSeeds, true);

code_string += Parser::Unparse(create_entity_code, &entity->evaluableNodeManager,
asset_params.prettyPrint, true, asset_params.sortKeys, false, 1);

entity->evaluableNodeManager.FreeNodeTree(create_entity_code);
}

code_string += Parser::transactionTermination;

bool all_stored_successfully = false;

if(asset_params.resourceType == FILE_EXTENSION_AMALGAM || asset_params.resourceType == FILE_EXTENSION_AMLG_METADATA)
{
std::ofstream outf(asset_params.resource, std::ios::out | std::ios::binary);
if(outf.good())
{
outf.write(code_string.c_str(), code_string.size());
outf.close();
all_stored_successfully = true;
}
}
else if(asset_params.resourceType == FILE_EXTENSION_COMPRESSED_AMALGAM_CODE)
{
//transform into format needed for compression
CompactHashMap<std::string, size_t> string_map;
string_map[code_string] = 0;

//compress and store
BinaryData compressed_data = CompressStrings(string_map);
all_stored_successfully = StoreFileFromBuffer<BinaryData>(asset_params.resource, asset_params.resourceType, compressed_data);
}

return all_stored_successfully;
}

//Stores an entity, including contained entities, etc. from the resource specified
// if update_persistence is true, then it will consider the persistent parameter, otherwise it is ignored
// if persistent is true, then it will keep the resource updated, if false it will clear persistence
Expand Down Expand Up @@ -158,13 +211,7 @@ class AssetManager
&& (asset_params.resourceType == FILE_EXTENSION_AMALGAM
|| asset_params.resourceType == FILE_EXTENSION_COMPRESSED_AMALGAM_CODE))
{
EvaluableNodeReference flattened_entity = EntityManipulation::FlattenEntity(&entity->evaluableNodeManager,
entity, *all_contained_entities, asset_params.includeRandSeeds, asset_params.parallelCreate);

bool all_stored_successfully = StoreResource(flattened_entity,
asset_params, &entity->evaluableNodeManager);

entity->evaluableNodeManager.FreeNodeTreeIfPossible(flattened_entity);
bool all_stored_successfully = FlattenAndStoreEntityToResource(entity, asset_params, *all_contained_entities);

if(update_persistence)
SetEntityPersistenceForFlattenedEntity(entity, persistent ? &asset_params : nullptr);
Expand Down
62 changes: 33 additions & 29 deletions src/Amalgam/Parser.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -131,17 +131,19 @@ std::tuple<EvaluableNodeReference, std::vector<std::string>, size_t> Parser::Par
}

std::string Parser::Unparse(EvaluableNode *tree, EvaluableNodeManager *enm,
bool expanded_whitespace, bool emit_attributes, bool sort_keys)
bool expanded_whitespace, bool emit_attributes, bool sort_keys,
bool first_of_transactional_unparse, size_t starting_indendation)
{
UnparseData upd;
upd.enm = enm;
upd.topNodeIfTransactionUnparsing = (first_of_transactional_unparse ? tree : nullptr);
//if the top node needs cycle checks, then need to check all nodes in case there are
// multiple ways to get to one
upd.cycleFree = (tree == nullptr || !tree->GetNeedCycleCheck());
upd.preevaluationNeeded = false;
upd.emitAttributes = emit_attributes;
upd.sortKeys = sort_keys;
Unparse(upd, tree, nullptr, expanded_whitespace, 0, false);
Unparse(upd, tree, nullptr, expanded_whitespace, starting_indendation, starting_indendation > 0);
return upd.result;
}

Expand Down Expand Up @@ -921,11 +923,6 @@ void Parser::Unparse(UnparseData &upd, EvaluableNode *tree, EvaluableNode *paren
return;
}

//if already hit this node, then need to create code to rebuild the circular reference

//add to check for circular references
upd.parentNodes[tree] = parent;

if(upd.emitAttributes)
{
AppendComments(tree, indentation_depth, expanded_whitespace, upd.result);
Expand Down Expand Up @@ -1095,34 +1092,41 @@ void Parser::Unparse(UnparseData &upd, EvaluableNode *tree, EvaluableNode *paren
}
}

//add closing parenthesis
if(expanded_whitespace)
if(tree != upd.topNodeIfTransactionUnparsing)
{
//indent if appropriate
if(recurse_expanded_whitespace)
//add closing parenthesis
if(expanded_whitespace)
{
for(size_t i = 0; i < indentation_depth; i++)
upd.result.push_back(indentationCharacter);
}
//indent if appropriate
if(recurse_expanded_whitespace)
{
for(size_t i = 0; i < indentation_depth; i++)
upd.result.push_back(indentationCharacter);
}

if(tree_type == ENT_LIST)
upd.result.push_back(']');
else if(tree_type == ENT_ASSOC)
upd.result.push_back('}');
else
upd.result.push_back(')');
if(tree_type == ENT_LIST)
upd.result.push_back(']');
else if(tree_type == ENT_ASSOC)
upd.result.push_back('}');
else
upd.result.push_back(')');

upd.result.push_back('\r');
upd.result.push_back('\n');
upd.result.push_back('\r');
upd.result.push_back('\n');
}
else
{
if(tree_type == ENT_LIST)
upd.result.push_back(']');
else if(tree_type == ENT_ASSOC)
upd.result.push_back('}');
else
upd.result.push_back(')');
}
}
else
else //end of opening transactional; emit a space to ensure things don't get improperly joined
{
if(tree_type == ENT_LIST)
upd.result.push_back(']');
else if(tree_type == ENT_ASSOC)
upd.result.push_back('}');
else
upd.result.push_back(')');
upd.result.push_back(' ');
}
}
}
Expand Down
15 changes: 13 additions & 2 deletions src/Amalgam/Parser.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -127,10 +127,17 @@ class Parser

//Returns a string that represents the tree
// if expanded_whitespace, will emit additional whitespace to make it easier to read
// if emit_attributes, then it will emit comments, labels, concurrency, preevaluations, etc.; if emit_attributes is false, then it will only emit values
// if emit_attributes, then it will emit comments, labels, concurrency, preevaluations, etc.;
// if emit_attributes is false, then it will only emit values
// if sort_keys, then it will perform a sort on all unordered nodes
// if first_of_transactional_unparse, it will not emit the final closing parenthesis or appropriate other character
// starting_indentation indicates where it will start, in case there was other code prior to which it is being concatenated
static std::string Unparse(EvaluableNode *tree, EvaluableNodeManager *enm,
bool expanded_whitespace = true, bool emit_attributes = true, bool sort_keys = false);
bool expanded_whitespace = true, bool emit_attributes = true, bool sort_keys = false,
bool first_of_transactional_unparse = false, size_t starting_indendation = 0);

//string to be appended after Unparse calls when the first one is called with first_of_transactional_unparse
inline static const std::string transactionTermination = ")";

//prefix used in the comments when attributing sources to EvaluableNodes
inline static const std::string sourceCommentPrefix = "src: ";
Expand All @@ -149,6 +156,10 @@ class Parser

EvaluableNodeManager *enm;

//if transactional unparsing, then this will be the top node
//if not, it will be nullptr
EvaluableNode *topNodeIfTransactionUnparsing;

//if true, then the tree is cycle free and don't need to keep track of potential circular references
bool cycleFree;

Expand Down
152 changes: 152 additions & 0 deletions src/Amalgam/entity/EntityManipulation.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -648,6 +648,158 @@ Entity *EntityManipulation::MutateEntity(Interpreter *interpreter, Entity *entit
return new_entity;
}

EvaluableNode *EntityManipulation::FlattenOnlyTopEntity(EvaluableNodeManager *enm, Entity *entity,
bool include_rand_seeds, bool ensure_en_flags_correct)
{
//////////
//build code to look like:
// (declare (assoc new_entity (null) create_new_entity (true))
// (let (assoc _ (lambda *entity code*))
// (if create_new_entity
// (assign "new_entity" (first
// (create_entities new_entity _)
// ))
// (assign_entity_roots new_entity _)
// )
// )
//
// [if include_rand_seeds]
// (set_entity_rand_seed
// new_entity
// *rand seed string* )
//
// [for each contained entity specified by the list representing the relative location to new_entity]
// [if parallel_create, will group these in ||(parallel ...) by container entity
//
// [if include_rand_seeds]
// (set_entity_rand_seed
// (first
// [always]
// (create_entities
// (append new_entity *relative id*)
// (lambda *entity code*) )
// (append new_entity *relative id*)
// *rand seed string* )
// [if include_rand_seeds]
// )
// *rand seed string* )
// )
// )

// (declare (assoc new_entity (null) create_new_entity (true))
EvaluableNode *declare_flatten = enm->AllocNode(ENT_DECLARE);

EvaluableNode *flatten_params = enm->AllocNode(ENT_ASSOC);
declare_flatten->AppendOrderedChildNode(flatten_params);
flatten_params->SetMappedChildNode(GetStringIdFromBuiltInStringId(ENBISI_new_entity), nullptr);
flatten_params->SetMappedChildNode(GetStringIdFromBuiltInStringId(ENBISI_create_new_entity), enm->AllocNode(ENT_TRUE));

// (let (assoc _ (lambda *entity code*))
EvaluableNode *let_entity_code = enm->AllocNode(ENT_LET);
declare_flatten->AppendOrderedChildNode(let_entity_code);
EvaluableNode *let_assoc = enm->AllocNode(ENT_ASSOC);
let_entity_code->AppendOrderedChildNode(let_assoc);

EvaluableNode *lambda_for_create_root = enm->AllocNode(ENT_LAMBDA);
let_assoc->SetMappedChildNode(GetStringIdFromBuiltInStringId(ENBISI__), lambda_for_create_root);

EvaluableNodeReference root_copy = entity->GetRoot(enm, EvaluableNodeManager::ENMM_LABEL_ESCAPE_INCREMENT);
lambda_for_create_root->AppendOrderedChildNode(root_copy);

// (if create_new_entity
EvaluableNode *if_create_new = enm->AllocNode(ENT_IF);
let_entity_code->AppendOrderedChildNode(if_create_new);
if_create_new->AppendOrderedChildNode(enm->AllocNode(ENT_SYMBOL, GetStringIdFromBuiltInStringId(ENBISI_create_new_entity)));

// (assign "new_entity" (first
// (create_entities new_entity _)
// ))
EvaluableNode *assign_new_entity_from_create = enm->AllocNode(ENT_ASSIGN);
if_create_new->AppendOrderedChildNode(assign_new_entity_from_create);
assign_new_entity_from_create->AppendOrderedChildNode(enm->AllocNode(ENT_STRING, GetStringIdFromBuiltInStringId(ENBISI_new_entity)));
EvaluableNode *create_root_entity = enm->AllocNode(ENT_CREATE_ENTITIES);
create_root_entity->AppendOrderedChildNode(enm->AllocNode(ENT_SYMBOL, GetStringIdFromBuiltInStringId(ENBISI_new_entity)));
create_root_entity->AppendOrderedChildNode(enm->AllocNode(ENT_SYMBOL, GetStringIdFromBuiltInStringId(ENBISI__)));
EvaluableNode *first_of_create_entity = enm->AllocNode(ENT_FIRST);
first_of_create_entity->AppendOrderedChildNode(create_root_entity);
assign_new_entity_from_create->AppendOrderedChildNode(first_of_create_entity);

// (assign_entity_roots new_entity _)
EvaluableNode *assign_new_entity_into_current = enm->AllocNode(ENT_ASSIGN_ENTITY_ROOTS);
if_create_new->AppendOrderedChildNode(assign_new_entity_into_current);
assign_new_entity_into_current->AppendOrderedChildNode(enm->AllocNode(ENT_SYMBOL, GetStringIdFromBuiltInStringId(ENBISI_new_entity)));
assign_new_entity_into_current->AppendOrderedChildNode(enm->AllocNode(ENT_SYMBOL, GetStringIdFromBuiltInStringId(ENBISI__)));

if(include_rand_seeds)
{
// (set_entity_rand_seed
// new_entity
// *rand seed string* )
EvaluableNode *set_rand_seed_root = enm->AllocNode(ENT_SET_ENTITY_RAND_SEED);
set_rand_seed_root->AppendOrderedChildNode(enm->AllocNode(ENT_SYMBOL, GetStringIdFromBuiltInStringId(ENBISI_new_entity)));
set_rand_seed_root->AppendOrderedChildNode(enm->AllocNode(ENT_STRING, entity->GetRandomState()));

declare_flatten->AppendOrderedChildNode(set_rand_seed_root);
}

if(root_copy.GetNeedCycleCheck())
{
if(ensure_en_flags_correct)
EvaluableNodeManager::UpdateFlagsForNodeTree(declare_flatten);
else //just set top node to inform whether it has cycles for future checks
declare_flatten->SetNeedCycleCheck(true);
}

return declare_flatten;
}

EvaluableNode *EntityManipulation::FlattenOnlyOneContainedEntity(EvaluableNodeManager *enm, Entity *entity, Entity *from_entity,
bool include_rand_seeds, bool ensure_en_flags_correct)
{
// (create_entities
// (append new_entity *relative id*)
// (lambda *entity code*)
// )
EvaluableNode *create_entity = enm->AllocNode(ENT_CREATE_ENTITIES);

EvaluableNode *src_id_list = GetTraversalIDPathFromAToB(enm, from_entity, entity);
EvaluableNode *src_append = enm->AllocNode(ENT_APPEND);
src_append->AppendOrderedChildNode(enm->AllocNode(ENT_SYMBOL, GetStringIdFromBuiltInStringId(ENBISI_new_entity)));
src_append->AppendOrderedChildNode(src_id_list);
create_entity->AppendOrderedChildNode(src_append);

EvaluableNode *lambda_for_create = enm->AllocNode(ENT_LAMBDA);
create_entity->AppendOrderedChildNode(lambda_for_create);

EvaluableNodeReference contained_root_copy = entity->GetRoot(enm, EvaluableNodeManager::ENMM_LABEL_ESCAPE_INCREMENT);
lambda_for_create->AppendOrderedChildNode(contained_root_copy);

if(include_rand_seeds)
{
// (set_entity_rand_seed
// (first ...create_entity... )
// *rand seed string* )
EvaluableNode *set_rand_seed = enm->AllocNode(ENT_SET_ENTITY_RAND_SEED);
EvaluableNode *first = enm->AllocNode(ENT_FIRST);
set_rand_seed->AppendOrderedChildNode(first);
first->AppendOrderedChildNode(create_entity);
set_rand_seed->AppendOrderedChildNode(enm->AllocNode(ENT_STRING, entity->GetRandomState()));

//replace the old create_entity with the one surrounded by setting rand seed
create_entity = set_rand_seed;
}

if(contained_root_copy.GetNeedCycleCheck())
{
if(ensure_en_flags_correct)
EvaluableNodeManager::UpdateFlagsForNodeTree(create_entity);
else //just set top node to inform whether it has cycles for future checks
create_entity->SetNeedCycleCheck(true);
}

return create_entity;
}

void EntityManipulation::SortEntitiesByID(std::vector<Entity *> &entities)
{
//for performance reasons, it may be worth considering other data structures if sort ever becomes or remains significant
Expand Down
Loading

0 comments on commit 5c6a9f1

Please sign in to comment.