world compression

data/maps/navigation_stress_test/bot.lua

@@ -1,12 +1,13 @@
 function goToRandomPoint()
+	local speed = math.random(2, 5)
 	local dst = {
 		math.random(-100, 100), 
 		0, 
 		math.random(-100, 100)
 	}
-	this.navmesh_agent:navigate(dst, 3, 0.5)
+	this.navmesh_agent:navigate(dst, speed, 0.5)
 	local speed_input = this.animator:getInputIndex("speed_y")
-	this.animator:setFloatInput(speed_input, 3)
+	this.animator:setFloatInput(speed_input, speed)
 end
 
 function start()

src/editor/asset_compiler.cpp

@@ -18,7 +18,6 @@
 #include "engine/engine.h"
 #include "engine/resource_manager.h"
 #include "engine/resource.h"
-#include <lz4/lz4.h>
 
 namespace Lumix {
 
@@ -85,7 +84,6 @@ struct AssetCompilerImpl : AssetCompiler {
 
 	~AssetCompilerImpl()
 	{
-		m_allocator.deallocate(m_lz4_state);
 		os::OutputFile file;
 		FileSystem& fs = m_app.getEngine().getFileSystem();
 		if (fs.open(".lumix/resources/_resources.txt_tmp", file)) {
@@ -210,24 +208,14 @@ struct AssetCompilerImpl : AssetCompiler {
 
 	bool writeCompiledResource(const Path& path, Span<const u8> data) override {
 		PROFILE_FUNCTION();
-		jobs::enter(&m_lz4_mutex);
 		constexpr u32 COMPRESSION_SIZE_LIMIT = 4096;
 		OutputMemoryStream compressed(m_allocator);
-		i32 compressed_size = 0;
 		if (data.length() > COMPRESSION_SIZE_LIMIT) {
-			if (!m_lz4_state) {
-				m_lz4_state = (u8*)m_allocator.allocate(LZ4_sizeofState(), 8);
-			}
-			const i32 cap = LZ4_compressBound((i32)data.length());
-			compressed.resize(cap);
-			compressed_size = LZ4_compress_fast_extState(m_lz4_state, (const char*)data.begin(), (char*)compressed.getMutableData(), (i32)data.length(), cap, 1); 
-			if (compressed_size == 0) {
+			if (!m_app.getEngine().compress(data, compressed)) {
 				logError("Could not compress ", path);
 				return false;
 			}
-			compressed.resize(compressed_size);
 		}
-		jobs::exit(&m_lz4_mutex);
 
 		FileSystem& fs = m_app.getEngine().getFileSystem();
 		const Path out_path(".lumix/resources/", path.getHash().getHashValue(), ".res");
@@ -238,6 +226,7 @@ struct AssetCompilerImpl : AssetCompiler {
 		}
 		CompiledResourceHeader header;
 		header.decompressed_size = data.length();
+		const u32 compressed_size = (u32)compressed.size();
 		if (data.length() > COMPRESSION_SIZE_LIMIT && compressed_size < i32(data.length() / 4 * 3)) {
 			header.flags |= CompiledResourceHeader::COMPRESSED;
 			(void)file.write(&header, sizeof(header));
@@ -829,8 +818,6 @@ struct AssetCompilerImpl : AssetCompiler {
 	DelegateList<void(Resource&, bool)> m_resource_compiled;
 	bool m_init_finished = false;
 	Array<Resource*> m_on_init_load;
-	u8* m_lz4_state = nullptr;
-	jobs::Mutex m_lz4_mutex;
 
 	u32 m_compile_batch_count = 0;
 	u32 m_batch_remaining_count = 0;

src/engine/engine.cpp

@@ -18,6 +18,7 @@
 #include "engine/prefab.h"
 #include "engine/resource_manager.h"
 #include "engine/world.h"
+#include <lz4/lz4.h>
 
 namespace Lumix
 {
@@ -50,7 +51,7 @@ struct EngineImpl final : Engine {
 		: m_allocator(allocator, "engine")
 		, m_page_allocator(allocator)
 		, m_prefab_resource_manager(m_allocator)
-		, m_resource_manager(m_allocator)
+		, m_resource_manager(*this, m_allocator)
 		, m_is_game_running(false)
 		, m_smooth_time_delta(1/60.f)
 		, m_time_multiplier(1.0f)
@@ -113,6 +114,8 @@ struct EngineImpl final : Engine {
 				logInfo(plugin_name, " plugin has not been loaded");
 			}
 		}
+
+		m_lz4_state = (u8*)m_allocator.allocate(LZ4_sizeofState(), 8);
 	}
 
 	void setMainWindow(os::WindowHandle wnd) override {
@@ -125,6 +128,8 @@ struct EngineImpl final : Engine {
 
 	~EngineImpl()
 	{
+		m_allocator.deallocate(m_lz4_state);
+
 		for (ISystem* system : m_system_manager->getSystems()) {
 			system->shutdownStarted();
 		}
@@ -243,6 +248,21 @@ struct EngineImpl final : Engine {
 		m_next_frame = true;
 	}
 
+	bool decompress(Span<const u8> src, Span<u8> output) override {
+		const i32 result = LZ4_decompress_safe((const char*)src.begin(), (char*)output.begin(), src.length(), output.length());
+		return result == output.length();
+	}
+
+	bool compress(Span<const u8> mem, OutputMemoryStream& output) override {
+		const i32 cap = LZ4_compressBound(mem.length());
+		const u32 start_size = (u32)output.size();
+		output.resize(cap + start_size);
+		jobs::MutexGuard guard(m_lz4_mutex);
+		const i32 compressed_size = LZ4_compress_fast_extState(m_lz4_state, (const char*)mem.begin(), (char*)output.getMutableData() + start_size, mem.length(), cap, 1); 
+		if (compressed_size == 0) return false;
+		output.resize(compressed_size + start_size);
+		return true;
+	}
 
 	void setTimeMultiplier(float multiplier) override
 	{
@@ -414,6 +434,8 @@ struct EngineImpl final : Engine {
 	os::WindowHandle m_window_handle = os::INVALID_WINDOW;
 	os::OutputFile m_log_file;
 	bool m_is_log_file_open = false;
+	u8* m_lz4_state = nullptr;
+	jobs::Mutex m_lz4_mutex;
 };
 
 

src/engine/engine.h

@@ -62,6 +62,8 @@ struct LUMIX_ENGINE_API Engine {
 	virtual void pause(bool pause) = 0;
 	virtual bool isPaused() const = 0;
 	virtual void nextFrame() = 0;
+	virtual bool decompress(Span<const u8> src, Span<u8> dst) = 0;
+	virtual bool compress(Span<const u8> src, OutputMemoryStream& dst) = 0;
 
 protected:
 	Engine() {}

src/engine/resource.cpp

@@ -5,13 +5,10 @@
 #include "core/path.h"
 #include "core/stream.h"
 #include "core/string.h"
-
+#include "engine/engine.h"
 #include "engine/resource.h"
 #include "engine/resource_manager.h"
 
-#include "lz4/lz4.h"
-
-
 namespace Lumix
 {
 
@@ -143,8 +140,9 @@ void Resource::fileLoaded(Span<const u8> blob, bool success) {
 	else if (header->flags & CompiledResourceHeader::COMPRESSED) {
 		OutputMemoryStream tmp(m_resource_manager.m_allocator);
 		tmp.resize(header->decompressed_size);
-		const i32 res = LZ4_decompress_safe((const char*)blob.begin() + sizeof(*header), (char*)tmp.getMutableData(), i32(blob.length() - sizeof(*header)), (i32)tmp.size());
-		if (res != header->decompressed_size || !load(tmp)) {
+		Engine& engine = m_resource_manager.getOwner().getEngine();
+		const Span<const u8> src((const u8*)blob.begin() + sizeof(*header), u64(blob.length() - sizeof(*header)));
+		if (!engine.decompress(src, Span<u8>(tmp.getMutableData(), tmp.size())) || !load(tmp)) {
 			++m_failed_dep_count;
 		}
 	}

src/engine/resource_manager.cpp

@@ -137,8 +137,9 @@ ResourceManager::~ResourceManager()
 	ASSERT(m_resources.empty());
 }
 
-ResourceManagerHub::ResourceManagerHub(IAllocator& allocator) 
+ResourceManagerHub::ResourceManagerHub(Engine& engine, IAllocator& allocator) 
 	: m_resource_managers(allocator)
+	, m_engine(engine)
 	, m_allocator(allocator)
 	, m_load_hook(nullptr)
 	, m_file_system(nullptr)

src/engine/resource_manager.h

@@ -6,9 +6,7 @@
 #include "core/hash_map.h"
 
 
-namespace Lumix
-{
-
+namespace Lumix {
 
 struct LUMIX_ENGINE_API ResourceManager {
 	friend struct Resource;
@@ -55,12 +53,13 @@ struct LUMIX_ENGINE_API ResourceManagerHub {
 		void continueLoad(Resource& res, bool success);
 	};
 
-	explicit ResourceManagerHub(IAllocator& allocator);
+	explicit ResourceManagerHub(struct Engine& engine, IAllocator& allocator);
 	~ResourceManagerHub();
 	ResourceManagerHub(const ResourceManagerHub& rhs) = delete;
 
 	void init(struct FileSystem& fs);
 
+	Engine& getEngine() { return m_engine; }
 	IAllocator& getAllocator() { return m_allocator; }
 	ResourceManager* get(ResourceType type);
 	const ResourceManagerTable& getAll() const { return m_resource_managers; }
@@ -93,6 +92,7 @@ struct LUMIX_ENGINE_API ResourceManagerHub {
 	IAllocator& m_allocator;
 	ResourceManagerTable m_resource_managers;
 	FileSystem* m_file_system;
+	Engine& m_engine;
 	LoadHook* m_load_hook;
 };
 

src/engine/world.cpp

@@ -836,65 +836,75 @@ void World::serialize(OutputMemoryStream& serializer, WorldSerializeFlags flags)
 	serializer.write(header);
 	serializeModuleList(*this, serializer);
 	serializer.write(flags);
-	serializer.write((u32)m_entities.size());
+
+	OutputMemoryStream blob(m_allocator);
+	blob.write((u32)m_entities.size());
 
 	for (u32 i = 0, c = m_entities.size(); i < c; ++i) {
 		if (!m_entities[i].valid) continue;
 		const EntityRef e = {(i32)i};
-		serializer.write(e);
-		serializer.write(m_transforms[i].pos);
-		serializer.write(m_transforms[i].rot);
-		serializer.write(m_transforms[i].scale);
-		if (serialize_partitions) serializer.write(m_entities[i].partition);
+		blob.write(e);
+		blob.write(m_transforms[i].pos);
+		blob.write(m_transforms[i].rot);
+		blob.write(m_transforms[i].scale);
+		if (serialize_partitions) blob.write(m_entities[i].partition);
 	}
-	serializer.write(INVALID_ENTITY);
+	blob.write(INVALID_ENTITY);
 
-	serializer.write((u32)m_names.size());
+	blob.write((u32)m_names.size());
 	for (const EntityName& name : m_names) {
-		serializer.write(name.entity);
-		serializer.writeString(name.name);
+		blob.write(name.entity);
+		blob.writeString(name.name);
 	}
 
-	serializer.write((u32)m_hierarchy.size());
+	blob.write((u32)m_hierarchy.size());
 	if (!m_hierarchy.empty()) {
 		for (const Hierarchy& h : m_hierarchy) {
-			serializer.write(h.entity);
-			serializer.write(h.parent);
-			serializer.write(h.first_child);
-			serializer.write(h.next_sibling);
-			serializer.write(h.local_transform.pos);
-			serializer.write(h.local_transform.rot);
-			serializer.write(h.local_transform.scale);
+			blob.write(h.entity);
+			blob.write(h.parent);
+			blob.write(h.first_child);
+			blob.write(h.next_sibling);
+			blob.write(h.local_transform.pos);
+			blob.write(h.local_transform.rot);
+			blob.write(h.local_transform.scale);
 		}
 	}
 
-	serializer.write((i32)m_modules.size());
+	blob.write((i32)m_modules.size());
 	for (const UniquePtr<IModule>& module : m_modules) {
-		serializer.writeString(module->getName());
-		serializer.write(module->getVersion());
-		module->serialize(serializer);
+		blob.writeString(module->getName());
+		blob.write(module->getVersion());
+		module->serialize(blob);
 	}
 
 	if (serialize_partitions) {
-		serializer.write((u32)m_partitions.size());
-		serializer.write(m_partitions.begin(), m_partitions.byte_size());
-		serializer.write(m_active_partition);
+		blob.write((u32)m_partitions.size());
+		blob.write(m_partitions.begin(), m_partitions.byte_size());
+		blob.write(m_active_partition);
 	}
+
+	const u64 offset = serializer.size();
+	serializer.write((u32)0);
+	serializer.write((u32)0);
+	m_engine.compress(blob, serializer);
+	u32* sizes = (u32*)(serializer.getMutableData() + offset);
+	sizes[0] = (u32)blob.size(); // uncompressed size
+	sizes[1] = u32(serializer.size() - offset - sizeof(u32) * 2); // compressed size
 }
 
-bool World::deserialize(InputMemoryStream& serializer, EntityMap& entity_map, WorldVersion& version)
+bool World::deserialize(InputMemoryStream& input, EntityMap& entity_map, WorldVersion& version)
 {
 	WorldHeader header;
 	WorldHeaderLegacy::Version legacy_version = WorldHeaderLegacy::Version::LAST;
-	serializer.read(header);
+	input.read(header);
 	if (header.magic == WorldEditorHeaderLegacy::MAGIC || header.magic == 0xffFFffFF) {
 		header.magic = WorldHeader::MAGIC;
 		// WorldEditorHeaderLegacy::Version matches first values of WorldHeaderVersion, so we can just use header.version as is
 		static_assert(sizeof(WorldEditorHeaderLegacy) == sizeof(WorldHeader));
-		serializer.read<u64>(); // hash
+		input.read<u64>(); // hash
 		WorldHeaderLegacy legacy_header;
-		serializer.read(legacy_header);
-		if (serializer.hasOverflow() || legacy_header.magic != WorldHeaderLegacy::MAGIC) {
+		input.read(legacy_header);
+		if (input.hasOverflow() || legacy_header.magic != WorldHeaderLegacy::MAGIC) {
 			logError("Wrong or corrupted file");
 			return false;
 		}
@@ -908,23 +918,36 @@ bool World::deserialize(InputMemoryStream& serializer, EntityMap& entity_map, Wo
 
 	version = header.version;
 
-	if (serializer.hasOverflow() || header.magic != WorldHeader::MAGIC) {
+	if (input.hasOverflow() || header.magic != WorldHeader::MAGIC) {
 		logError("Wrong or corrupted file");
 		return false;
 	}
 	if (header.version > WorldVersion::LATEST) {
 		logError("Unsupported version of world");
 		return false;
 	}
-	if (!hasSerializedModules(*this, serializer)) return false;
+	if (!hasSerializedModules(*this, input)) return false;
 
 	bool deserialize_partitions = false;
 	if (legacy_version > WorldHeaderLegacy::Version::FLAGS) {
 		WorldSerializeFlags flags;
-		serializer.read(flags);
+		input.read(flags);
 		deserialize_partitions = (u32)flags & (u32)WorldSerializeFlags::HAS_PARTITIONS;
 	}
-
+
+	InputMemoryStream serializer(input.skip(0), input.remaining());
+	OutputMemoryStream uncompressed(m_allocator);
+	if (header.version > WorldVersion::COMPRESSED) { 
+		u32 uncompressed_size;
+		u32 compressed_size;
+		input.read(uncompressed_size);
+		input.read(compressed_size);
+		uncompressed.resize(uncompressed_size);
+		m_engine.decompress(Span((const u8*)input.skip(0), compressed_size), Span(uncompressed.getMutableData(), uncompressed.size()));
+		serializer = InputMemoryStream(uncompressed);
+		input.skip(compressed_size);
+	}
+
 	u32 to_reserve;
 	serializer.read(to_reserve);
 	entity_map.reserve(to_reserve);
@@ -1008,6 +1031,10 @@ bool World::deserialize(InputMemoryStream& serializer, EntityMap& entity_map, Wo
 		logError("End of file encountered while trying to read data");
 		return false;
 	}
+
+	if (header.version <= WorldVersion::COMPRESSED) { 
+		input.skip(serializer.getPosition());
+	}
 	return true;
 }
 

src/engine/world.h

@@ -20,13 +20,14 @@ enum class WorldVersion : u32 {
 	HASH64,
 	NEW_ENTITY_FOLDERS,
 	MERGED_HEADERS,
+	COMPRESSED,
 
 	LATEST
 };
 
 enum class WorldSerializeFlags : u32 { 
 	HAS_PARTITIONS = 1 << 0,
-	
+
 	NONE = 0
 };