18866: Adds header to CAML file, MAJOR (#60)

* Amalgam CAML files now have a header:
  - 4 bytes - magic number:  'c' 'a' 'm' 'l'
  - uint32_t - major version
  - uint32_t - minor version
  - uint32_t - patch version
  - Any previously created CAML files are now unsupported
* Updates LoadEntity to have richer status reporting when files cannot
be read and adds VerifyEntity that verifies entity from a file but
doesn't load it
* Removes CSTL (compressed strings) file support
* Adds `amlg_code` directory to files list
* Cleans-up C++ header includes
* Cleans-up test output files and moves REPL to examples folder

.gitignore

@@ -14,7 +14,8 @@ src/Amalgam/AmalgamVersion.h
 .envrc
 .env
 
-## Amalgam metadata files
+## Amalgam binary files
+*.caml
 *.mdam
 
 ## Build results

CMakeLists.txt

@@ -134,6 +134,8 @@ set(COMMON_SOURCE
     src/Amalgam/FilenameEscapeProcessor.h
     src/Amalgam/GeneralizedDistance.h
     src/Amalgam/HashMaps.h
+    src/Amalgam/importexport/FileSupportCAML.cpp
+    src/Amalgam/importexport/FileSupportCAML.h
     src/Amalgam/importexport/FileSupportCSV.cpp
     src/Amalgam/importexport/FileSupportCSV.h
     src/Amalgam/importexport/FileSupportJSON.cpp
@@ -176,8 +178,6 @@ set(COMMON_SOURCE
     src/Amalgam/string/StringInternPool.h
     src/Amalgam/string/StringManipulation.cpp
     src/Amalgam/string/StringManipulation.h
-    # Add out.txt so it shows up in IDE
-    src/Amalgam/out.txt
 )
 set(COMMON_SOURCE_THREADS
     ${COMMON_SOURCE}
@@ -206,6 +206,8 @@ source_group(TREE ${CMAKE_SOURCE_DIR} FILES ${AMALGAM_ALL_SOURCE})
 # Get all files we care about for easy access in projects (mainly IDEs)
 file(GLOB_RECURSE ALL_FILES "${CMAKE_SOURCE_DIR}/*")
 list(FILTER ALL_FILES EXCLUDE REGEX "(.*/.git/.*|.*/out/.*|.*/src/.*|.*/test/.*|.*Amalgam\.sln.*)")
+file(GLOB_RECURSE ADDITIONAL_FILES "${CMAKE_SOURCE_DIR}/src/Amalgam/out.txt" "${CMAKE_SOURCE_DIR}/src/Amalgam/amlg_code/*")
+list(APPEND ALL_FILES ${ADDITIONAL_FILES})
 source_group(TREE ${CMAKE_SOURCE_DIR} FILES ${ALL_FILES})
 list(APPEND AMALGAM_APP_ONLY_SOURCE ${ALL_FILES})
 

docs/language.js

@@ -1332,39 +1332,39 @@ var data = [
 		"parameter" : "load string file_path [bool escape_filename] [string file_type]",
 		"output" : "*",
 		"permissions" : "r",
-		"description" : "Loads the data specified by the resource in string.  Attempts to load the file type and parse it into appropriate data and evaluate to the corresponding code. The parameter escape_filename defaults to false, but if it is true, it will agressively escape filenames using only alphanumeric characters and the underscore, using underscore as an escape character.  If file_type is specified and not null, it will use the file_type specified instead of the extension of the file_path.  File formats supported are amlg, json, yaml, csv, cstl, and caml; anything not in this list will be loaded as a binary string.  Note that loading from a non-'.amlg' extension will only ever provide lists, assocs, numbers, and strings.",
+		"description" : "Loads the data specified by the resource in string.  Attempts to load the file type and parse it into appropriate data and evaluate to the corresponding code. The parameter escape_filename defaults to false, but if it is true, it will agressively escape filenames using only alphanumeric characters and the underscore, using underscore as an escape character.  If file_type is specified and not null, it will use the file_type specified instead of the extension of the file_path.  File formats supported are amlg, json, yaml, csv, and caml; anything not in this list will be loaded as a binary string.  Note that loading from a non-'.amlg' extension will only ever provide lists, assocs, numbers, and strings.",
 		"example" : "(print (load \"my_directory/MyModule.amlg\"))"
 	},
 
 	{
 		"parameter" : "load_entity string file_path [id entity] [bool escape_filename] [bool escape_contained_filenames] [string file_type]",
 		"output" : "id",
 		"permissions" : "r",
-		"description" : "Loads an entity specified by the resource in string.  Attempts to load the file type and parse it into appropriate data and store it in the entity specified by id, following the same id creation rules as create_entities, except that if no id is specified, it may default to a name based on the resource if available.  The parameter escape_filename defaults to false, but if it is true, it will agressively escape filenames using only alphanumeric characters and the underscore, using underscore as an escape character.  If escape_contained_filenames is true, which is its default, it will also escape contained entity filenames. If file_type is specified and not null, it will use the file_type specified instead of the extension of the file_path.  File formats supported are amlg, json, yaml, csv, cstl, and caml; anything not in this list will be loaded as a binary string.  Note that loading from a non-'.amlg' extension will only ever provide lists, assocs, numbers, and strings.",
+		"description" : "Loads an entity specified by the resource in string.  Attempts to load the file type and parse it into appropriate data and store it in the entity specified by id, following the same id creation rules as create_entities, except that if no id is specified, it may default to a name based on the resource if available.  The parameter escape_filename defaults to false, but if it is true, it will agressively escape filenames using only alphanumeric characters and the underscore, using underscore as an escape character.  If escape_contained_filenames is true, which is its default, it will also escape contained entity filenames. If file_type is specified and not null, it will use the file_type specified instead of the extension of the file_path.  File formats supported are amlg, json, yaml, csv, and caml; anything not in this list will be loaded as a binary string.  Note that loading from a non-'.amlg' extension will only ever provide lists, assocs, numbers, and strings.",
 		"example" : "(load_entity \"my_directory/MyModule.amlg\" \"MyModule\")"
 	},
 
 	{
 		"parameter" : "load_persistent_entity string file_path [id entity] [bool escape_filename]",
 		"output" : "id",
 		"permissions" : "r",
-		"description" : "Loads an entity specified by the resource in string.  Attempts to load the file type and parse it into appropriate data and store it in the entity specified by id, following the same id creation rules as create_entities. Any modifications to the entity or any entity contained within it will be written out to the resource, so that the memory and persistent storage are synchronized.  The parameter escape_filename defaults to false, but if it is true, it will agressively escape filenames using only alphanumeric characters and the underscore, using underscore as an escape character.  This command will escape contained filenames.  The file type of a persisted entity must match the extension of the file of the main entity.  File formats supported are amlg, json, yaml, csv, cstl, and caml; anything not in this list will be loaded as a binary string.  Note that loading from a non-'.amlg' extension will only ever provide lists, assocs, numbers, and strings.\n\n<b>WARNING:</b> Loading the same file as a persistent entity in more than one place will overwrite the file each time either entity is altered, but changes will not be propogated between the entities.",
+		"description" : "Loads an entity specified by the resource in string.  Attempts to load the file type and parse it into appropriate data and store it in the entity specified by id, following the same id creation rules as create_entities. Any modifications to the entity or any entity contained within it will be written out to the resource, so that the memory and persistent storage are synchronized.  The parameter escape_filename defaults to false, but if it is true, it will agressively escape filenames using only alphanumeric characters and the underscore, using underscore as an escape character.  This command will escape contained filenames.  The file type of a persisted entity must match the extension of the file of the main entity.  File formats supported are amlg, json, yaml, csv, and caml; anything not in this list will be loaded as a binary string.  Note that loading from a non-'.amlg' extension will only ever provide lists, assocs, numbers, and strings.\n\n<b>WARNING:</b> Loading the same file as a persistent entity in more than one place will overwrite the file each time either entity is altered, but changes will not be propogated between the entities.",
 		"example" : "(load_persistent_entity \"my_directory/MyModule.amlg\" \"MyModule\")"
 	},
 
 	{
 		"parameter" : "store string file_path * node [bool escape_filename] [string file_type] [assoc params]",
 		"output" : "bool",
 		"permissions" : "r",
-		"description" : "Stores the code specified by * to the resource in string. Returns true if successful, false if not. The parameter escape_filename defaults to false, but if it is true, it will agressively escape filenames using only alphanumeric characters and the underscore, using underscore as an escape character.   If file_type is specified and not null, it will use the file_type specified instead of the extension of the file_path.  File formats supported are amlg, json, yaml, csv, cstl, and caml; anything not in this list will be loaded as a binary string.  Note that loading from a non-'.amlg' extension will only ever provide lists, assocs, numbers, and strings.  If params is specified, it is an assoc that contains key-value pairs describing the format.  The key \"sort_keys\" can be used to specify a boolean value, if true, then it will sort the keys, otherwise the default behavior is to emit the keys based on memory layout.",
+		"description" : "Stores the code specified by * to the resource in string. Returns true if successful, false if not. The parameter escape_filename defaults to false, but if it is true, it will agressively escape filenames using only alphanumeric characters and the underscore, using underscore as an escape character.   If file_type is specified and not null, it will use the file_type specified instead of the extension of the file_path.  File formats supported are amlg, json, yaml, csv, and caml; anything not in this list will be loaded as a binary string.  Note that loading from a non-'.amlg' extension will only ever provide lists, assocs, numbers, and strings.  If params is specified, it is an assoc that contains key-value pairs describing the format.  The key \"sort_keys\" can be used to specify a boolean value, if true, then it will sort the keys, otherwise the default behavior is to emit the keys based on memory layout.",
 		"example" : "(store \"my_directory/MyData.amlg\" (list 1 2 3))"
 	},
 
 	{
 		"parameter" : "store_entity string file_path id entity [bool escape_filename] [bool escape_contained_filenames] [string file_type] [assoc params]",
 		"output" : "bool",
 		"permissions" : "r",
-		"description" : "Stores the entity specified by the id to the resource in string. Returns true if successful, false if not. The parameter escape_filename defaults to false, but if it is true, it will agressively escape filenames using only alphanumeric characters and the underscore, using underscore as an escape character.  If escape_contained_filenames is true, which is its default, it will also escape contained entity filenames.  If file_type is specified and not null, it will use the file_type specified instead of the extension of the file_path.  File formats supported are amlg, json, yaml, csv, cstl, and caml; anything not in this list will be loaded as a binary string.  Note that loading from a non-'.amlg' extension will only ever provide lists, assocs, numbers, and strings.  If params is specified, it is an assoc that contains key-value pairs describing the format.  The key \"sort_keys\" can be used to specify a boolean value, if true, then it will sort the keys, otherwise the default behavior is to emit the keys based on memory layout.",
+		"description" : "Stores the entity specified by the id to the resource in string. Returns true if successful, false if not. The parameter escape_filename defaults to false, but if it is true, it will agressively escape filenames using only alphanumeric characters and the underscore, using underscore as an escape character.  If escape_contained_filenames is true, which is its default, it will also escape contained entity filenames.  If file_type is specified and not null, it will use the file_type specified instead of the extension of the file_path.  File formats supported are amlg, json, yaml, csv, and caml; anything not in this list will be loaded as a binary string.  Note that loading from a non-'.amlg' extension will only ever provide lists, assocs, numbers, and strings.  If params is specified, it is an assoc that contains key-value pairs describing the format.  The key \"sort_keys\" can be used to specify a boolean value, if true, then it will sort the keys, otherwise the default behavior is to emit the keys based on memory layout.",
 		"example" : "(store_entity \"my_directory/MyData.amlg\" \"MyData\")"
 	},
 

examples/README.md

@@ -11,5 +11,6 @@ A collection of notable and neat/fun examples written in Amalgam located in dire
 
 * [Conway's Game of Life](https://en.wikipedia.org/wiki/Conway%27s_Game_of_Life)
 * [Fractals](https://en.wikipedia.org/wiki/Mandelbrot_set)
-* [Quine](https://en.wikipedia.org/wiki/Quine_(computing))
 * [Hello, World!](https://en.wikipedia.org/wiki/%22Hello,_World!%22_program)
+* [Quine](https://en.wikipedia.org/wiki/Quine_(computing))
+* [REPL](https://en.wikipedia.org/wiki/Read%E2%80%93eval%E2%80%93print_loop)

examples/json_search/json-combine.amlg

@@ -1,4 +1,4 @@
-#!./amalgam-mt
+#!amalgam-mt
 (seq
 	(declare (assoc
 		file_a (get argv 1)

examples/repl/repl.amlg

@@ -0,0 +1,7 @@
+(while (true)
+	(system "printline" "> ")
+	(let
+		(assoc input (system "readline") )
+		(print (call (parse input)) "\n")
+	)
+)

src/Amalgam/Amalgam.h

@@ -1,7 +1,8 @@
 #pragma once
 
 //system headers:
-#include <string>
+#include <cstddef>
+#include <cstdint>
 
 #if defined(_MSC_VER)
 //Microsoft 
@@ -15,8 +16,19 @@
 
 extern "C"
 {
+	//status from LoadEntity
+	struct LoadEntityStatus
+	{
+		bool loaded;
+		char *message;
+		char *version;
+	};
+
 	//loads the entity specified into handle
-	AMALGAM_EXPORT bool   LoadEntity(char *handle, char *path, bool persistent, bool load_contained_entities, char *write_log_filename, char *print_log_filename);
+	AMALGAM_EXPORT LoadEntityStatus LoadEntity(char *handle, char *path, bool persistent, bool load_contained_entities, char *write_log_filename, char *print_log_filename);
+
+	//verifies the entity specified by path. Uses LoadEntityStatus to return any errors and version
+	AMALGAM_EXPORT LoadEntityStatus VerifyEntity(char *path);
 
 	//stores the entity specified by handle into path
 	AMALGAM_EXPORT void   StoreEntity(char *handle, char *path, bool update_persistence_location = false, bool store_contained_entities = true);
@@ -37,7 +49,6 @@ extern "C"
 	AMALGAM_EXPORT void   AppendNumberValue(char *handle, char *label, double value);
 	AMALGAM_EXPORT void   SetNumberValue(char *handle, char *label, double value);
 
-	AMALGAM_EXPORT size_t PrepStringValueToTransferBuffer(char *handle, char *label);
 	AMALGAM_EXPORT void   AppendStringValue(char *handle, char *label, char *value);
 	AMALGAM_EXPORT void   SetStringValue(char *handle, char *label, char *value);
 
@@ -55,7 +66,7 @@ extern "C"
 	AMALGAM_EXPORT void   AppendStringList(char *handle, char *label, char **list, size_t len);
 	AMALGAM_EXPORT void   SetStringList(char *handle, char *label, char **list, size_t len);
 
-	AMALGAM_EXPORT void SetJSONToLabel(char *handle, char *label, char *json);
+	AMALGAM_EXPORT void   SetJSONToLabel(char *handle, char *label, char *json);
 
 	AMALGAM_EXPORT wchar_t *GetJSONPtrFromLabelWide(char *handle, char *label);
 	AMALGAM_EXPORT char *GetJSONPtrFromLabel(char *handle, char *label);

src/Amalgam/Amalgam.vcxproj

@@ -569,6 +569,7 @@
     <ClCompile Include="evaluablenode\EvaluableNodeTreeDifference.cpp" />
     <ClCompile Include="evaluablenode\EvaluableNodeTreeFunctions.cpp" />
     <ClCompile Include="evaluablenode\EvaluableNodeTreeManipulation.cpp" />
+    <ClCompile Include="importexport\FileSupportCAML.cpp" />
     <ClCompile Include="importexport\FileSupportCSV.cpp" />
     <ClCompile Include="importexport\FileSupportJSON.cpp" />
     <ClCompile Include="importexport\FileSupportYAML.cpp" />
@@ -634,6 +635,7 @@
     <ClInclude Include="FilenameEscapeProcessor.h" />
     <ClInclude Include="GeneralizedDistance.h" />
     <ClInclude Include="HashMaps.h" />
+    <ClInclude Include="importexport\FileSupportCAML.h" />
     <ClInclude Include="importexport\FileSupportCSV.h" />
     <ClInclude Include="importexport\FileSupportJSON.h" />
     <ClInclude Include="importexport\FileSupportYAML.h" />

src/Amalgam/Amalgam.vcxproj.filters

@@ -42,6 +42,9 @@
     <ClCompile Include="importexport\FileSupportCSV.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="importexport\FileSupportCAML.cpp">
+      <Filter>Source Files</Filter>
+    </ClCompile>
     <ClCompile Include="rand\RandomStream.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
@@ -191,6 +194,9 @@
     <ClInclude Include="importexport\FileSupportCSV.h">
       <Filter>Header Files</Filter>
     </ClInclude>
+    <ClInclude Include="importexport\FileSupportCAML.h">
+      <Filter>Header Files</Filter>
+    </ClInclude>
     <ClInclude Include="AmalgamVersion.h">
       <Filter>Header Files</Filter>
     </ClInclude>

src/Amalgam/AmalgamAPI.cpp

@@ -6,11 +6,6 @@
 #include "EntityQueries.h"
 
 //system headers:
-#include <algorithm>
-#include <cstdio>
-#include <cstring>
-#include <iostream>
-#include <map>
 #include <string>
 
 //Workaround because GCC doesn't support strcpy_s
@@ -77,18 +72,34 @@ extern "C"
 		return wct;
 	}
 
+	LoadEntityStatus ConvertLoadStatusToCStatus(EntityExternalInterface::LoadEntityStatus &status)
+	{
+		return {
+			status.loaded,
+			StringToCharPtr(status.message),
+			StringToCharPtr(status.version)
+		};
+	}
+
 	// ************************************
 	// api methods
 	// ************************************
 
-	bool LoadEntity(char *handle, char *path, bool persistent, bool load_contained_entities, char *write_log_filename, char *print_log_filename)
+	LoadEntityStatus LoadEntity(char *handle, char *path, bool persistent, bool load_contained_entities, char *write_log_filename, char *print_log_filename)
 	{
 		std::string h(handle);
 		std::string p(path);
 		std::string wlfname(write_log_filename);
 		std::string plfname(print_log_filename);
+		auto status = entint.LoadEntity(h, p, persistent, load_contained_entities, wlfname, plfname);
+		return ConvertLoadStatusToCStatus(status);
+	}
 
-		return entint.LoadEntity(h, p, persistent, load_contained_entities, wlfname, plfname);
+	LoadEntityStatus VerifyEntity(char *path)
+	{
+		std::string p(path);
+		auto status = entint.VerifyEntity(p);
+		return ConvertLoadStatusToCStatus(status);
 	}
 
 	void StoreEntity(char *handle, char *path, bool update_persistence_location, bool store_contained_entities)

src/Amalgam/AmalgamMain.cpp

@@ -229,10 +229,11 @@ PLATFORM_MAIN_CONSOLE
 	else
 	{
 		//run the standard amlg command line interface
+		EntityExternalInterface::LoadEntityStatus status;
 		std::string file_type = "";
-		Entity *entity = asset_manager.LoadEntityFromResourcePath(amlg_file_to_run, file_type, false, true, false, true, random_seed);
-		if(entity == nullptr)
-			return 0;
+		Entity *entity = asset_manager.LoadEntityFromResourcePath(amlg_file_to_run, file_type, false, true, false, true, random_seed, status);
+		if(!status.loaded)
+			return 1;
 
 		asset_manager.SetRootPermission(entity, true);
 

src/Amalgam/AmalgamTrace.cpp

@@ -1,13 +1,7 @@
 //project headers:
-#include "Amalgam.h"
 #include "AmalgamVersion.h"
 #include "AssetManager.h"
-#include "Entity.h"
 #include "EntityExternalInterface.h"
-#include "EntityQueries.h"
-#include "EvaluableNode.h"
-#include "Parser.h"
-#include "PerformanceProfiler.h"
 #include "PlatformSpecific.h"
 #include "RandomStream.h"
 
@@ -75,8 +69,8 @@ int32_t RunAmalgamTrace(std::istream *in_stream, std::ostream *out_stream, std::
 					transaction_listener_path = "";
 
 				std::string new_rand_seed = random_stream.CreateOtherStreamStateViaString("trace");
-				bool result = entint.LoadEntity(handle, data, persistent == "true", use_contained == "true", transaction_listener_path, print_listener_path, new_rand_seed);
-				response = result ? SUCCESS_RESPONSE : FAILURE_RESPONSE;
+				auto status = entint.LoadEntity(handle, data, persistent == "true", use_contained == "true", transaction_listener_path, print_listener_path, new_rand_seed);
+				response = status.loaded ? SUCCESS_RESPONSE : FAILURE_RESPONSE;
 			}
 			else
 			{