21695: Fixes WASM debug build, packaging updates (#274)

* Fixes issue where WASM debug build did not actually have debug
information available
* Adds `web` to `sENVIRONMENTS` linker flag
* Removes quiet mode from CI/CD build step for future debugability
* Splits build process into separate release/debug phases for WASM
instead of shoehorning a debug build into the release build

---------

Co-authored-by: howsohazard <143410553+howsohazard@users.noreply.github.com>
Co-authored-by: Lance Gliser <lance.gliser@howso.com>

.gitignore

@@ -1,6 +1,9 @@
 ## Use glob syntax
 syntax: glob
 
+## Local files
+/local
+
 ## Files created by build
 src/Amalgam/AmalgamVersion.h
 

.vscode/settings.json

@@ -1,4 +1,4 @@
 {
-  "cSpell.words": ["KullbackLeibler"],
-  "files.trimTrailingWhitespace": false,
+  "cSpell.words": ["amlg", "cwrap", "KullbackLeibler"],
+  "files.trimTrailingWhitespace": false
 }

CMakeLists.txt

@@ -313,15 +313,8 @@ if(IS_LINUX AND IS_AMD64)
         SOURCE ${AMALGAM_LIB_ONLY_SOURCE} ${COMMON_SOURCE_THREADS} IDE_FOLDER "OtherBuildTargets")
 endif()
 
-# WASM single-threaded debug target (amalgam-st-debug)
-if(IS_WASM)
-    add_compiled_target(AUTO_NAME TYPE "objlib" USE_DEBUG
-        SOURCE ${COMMON_SOURCE} IDE_FOLDER "OtherBuildTargets")
-    add_compiled_target(AUTO_NAME TYPE "app" USE_DEBUG
-        SOURCE ${COMMON_SOURCE} APP_ONLY_SOURCE ${AMALGAM_APP_ONLY_SOURCE})
-    add_compiled_target(AUTO_NAME TYPE "sharedlib" USE_DEBUG
-        SOURCE ${AMALGAM_LIB_ONLY_SOURCE} ${COMMON_SOURCE} IDE_FOLDER "OtherBuildTargets")
-endif()
+# Print the current build type
+message(STATUS "Current build type: ${CMAKE_BUILD_TYPE}")
 
 #
 # Additional artifacts/test/etc

CMakePresets.json

@@ -272,6 +272,16 @@
         "CMAKE_TRY_COMPILE_TARGET_TYPE": "STATIC_LIBRARY"
       }
     },
+    {
+      "name": "wasm64-debug-unknown",
+      "description": "emcc for wasm64 (debug)",
+      "inherits": [ "base", "wasm64", "debug", "emcc" ],
+      "cacheVariables": {
+        "CMAKE_OSX_ARCHITECTURES": "",
+        "CMAKE_CXX_COMPILER_TARGET": "wasm64-unknown-emscripten",
+        "CMAKE_TRY_COMPILE_TARGET_TYPE": "STATIC_LIBRARY"
+      }
+    },
 
     {
       "name": "amd64-debug-macos",
@@ -356,6 +366,11 @@
       "configurePreset": "wasm64-release-unknown",
       "description": "wasm64 release build"
     },
+    {
+      "name": "wasm64-debug-unknown",
+      "configurePreset": "wasm64-debug-unknown",
+      "description": "wasm64 debug build"
+    },
     {
       "name": "amd64-debug-macos",
       "configurePreset": "amd64-debug-macos",

README.md

@@ -114,16 +114,16 @@ Pre-built binaries use CMake+Ninja for CI/CD. See [PR workflow](.github/workflow
 
 Though Amalgam is intended to support any C++17 compliant compiler, the current specific tool and OS versions used are:
 
-* CMake 3.23
-* Ninja 1.11
+* CMake 3.30
+* Ninja 1.10
 * Windows:
     * Visual Studio 2022 v143
 * Linux:
     * Ubuntu 20.04, gcc-10
 * macOS (Darwin):
-    * macOS 12, AppleClang 14.0
+    * macOS 13, AppleClang 15.0
 * WASM:
-    * Ubuntu 20.04, emscripten 3.1.32
+    * Ubuntu 20.04, emscripten 3.1.67
 
 #### Runtime Requirements
 
@@ -148,7 +148,7 @@ Running the pre-built interpreter has specific runtime requirements per platform
 
 ##### Linux
 
-* glibc 2.29 or later
+* glibc 2.31 or later
 * Arch: amd64 or arm64
     * Specific arm64 builds: `armv8-a+simd` & `armv8.2-a+simd+rcpc`
 

build/cmake/custom_add_target.cmake

@@ -10,7 +10,7 @@ set(ALL_OBJLIB_TARGETS)
 set(ALL_SHAREDLIB_TARGETS)
 set(ALL_APP_TARGETS)
 function(add_compiled_target)
-    set(options AUTO_NAME USE_THREADS USE_OPENMP USE_PGC USE_AFMI_MT USE_AFMI_ST USE_DEBUG USE_ADVANCED_ARCH_INTRINSICS NO_INSTALL)
+    set(options AUTO_NAME USE_THREADS USE_OPENMP USE_PGC USE_AFMI_MT USE_AFMI_ST USE_ADVANCED_ARCH_INTRINSICS NO_INSTALL)
     set(oneValueArgs NAME TYPE OUTPUT_NAME_BASE IDE_FOLDER)
     set(multiValueArgs INCLUDE_DIRS COMPILER_DEFINES LINK_LIBRARIES SOURCE APP_ONLY_SOURCE)
     cmake_parse_arguments(args "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
@@ -29,7 +29,6 @@ function(add_compiled_target)
     set(USE_PGC ${args_USE_PGC})
     set(USE_AFMI_MT ${args_USE_AFMI_MT})
     set(USE_AFMI_ST ${args_USE_AFMI_ST})
-    set(USE_DEBUG ${args_USE_DEBUG})
     set(USE_ADVANCED_ARCH_INTRINSICS ${args_USE_ADVANCED_ARCH_INTRINSICS})
     set(NO_INSTALL ${args_NO_INSTALL})
 
@@ -93,7 +92,7 @@ function(add_compiled_target)
             string(APPEND TARGET_NAME_BASE "-mt-afmi")
         elseif(USE_AFMI_ST)
             string(APPEND TARGET_NAME_BASE "-st-afmi")
-        elseif(USE_DEBUG)
+        elseif(IS_WASM AND CMAKE_BUILD_TYPE STREQUAL "Debug")
             string(APPEND TARGET_NAME_BASE "-st-debug")
         else()
             string(APPEND TARGET_NAME_BASE "-st")
@@ -251,15 +250,6 @@ function(add_compiled_target)
         target_compile_options(${TARGET_NAME} PUBLIC -O1)
     endif()
 
-    if(USE_DEBUG)
-        target_compile_options(${TARGET_NAME} PUBLIC -g -O1)
-        target_compile_definitions(${TARGET_NAME} PUBLIC DEBUG_BUILD)
-        if (IS_WASM)
-            # Enabling WASM assertions can greatly assist in the debugging process
-            string(APPEND CMAKE_EXE_LINKER_FLAGS " -sASSERTIONS=2")
-        endif()
-    endif()
-
     # Advanced arch intrinsics:
     if(USE_ADVANCED_ARCH_INTRINSICS AND NOT IS_WASM)
         if (IS_AMD64)
@@ -290,13 +280,24 @@ function(add_compiled_target)
 
             # Extra files to install for WASM
             if(IS_WASM)
-                install(
-                    FILES
-                        "$<TARGET_FILE_DIR:${TARGET_NAME}>/$<TARGET_FILE_BASE_NAME:${TARGET_NAME}>.data"
-                        "$<TARGET_FILE_DIR:${TARGET_NAME}>/$<TARGET_FILE_BASE_NAME:${TARGET_NAME}>.wasm"
-                    DESTINATION "${INSTALL_DIR}"
-                    PERMISSIONS ${DEFAULT_INSTALL_PERMISSIONS}
-                )
+                if(CMAKE_BUILD_TYPE STREQUAL "Debug")
+                    install(
+                        FILES
+                            "$<TARGET_FILE_DIR:${TARGET_NAME}>/$<TARGET_FILE_BASE_NAME:${TARGET_NAME}>.data"
+                            "$<TARGET_FILE_DIR:${TARGET_NAME}>/$<TARGET_FILE_BASE_NAME:${TARGET_NAME}>.wasm"
+                            "$<TARGET_FILE_DIR:${TARGET_NAME}>/$<TARGET_FILE_BASE_NAME:${TARGET_NAME}>.wasm.debug.wasm"
+                        DESTINATION "${INSTALL_DIR}"
+                        PERMISSIONS ${DEFAULT_INSTALL_PERMISSIONS}
+                    )
+                else()
+                    install(
+                        FILES
+                            "$<TARGET_FILE_DIR:${TARGET_NAME}>/$<TARGET_FILE_BASE_NAME:${TARGET_NAME}>.data"
+                            "$<TARGET_FILE_DIR:${TARGET_NAME}>/$<TARGET_FILE_BASE_NAME:${TARGET_NAME}>.wasm"
+                        DESTINATION "${INSTALL_DIR}"
+                        PERMISSIONS ${DEFAULT_INSTALL_PERMISSIONS}
+                    )
+                endif()
                 file(MAKE_DIRECTORY "out/config")
                 set(WASM_DECLARATION_FILE "out/config/${TARGET_NAME_BASE}.d.cts")
                 file(COPY_FILE "build/wasm/amalgam-wasm.d.cts" "${WASM_DECLARATION_FILE}" ONLY_IF_DIFFERENT)

build/cmake/global_compiler_flags.cmake

@@ -75,7 +75,13 @@ elseif("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU" OR "${CMAKE_CXX_COMPILER_ID}" S
     # TODO 1599: WASM support is experimental, these flags will be cleaned up and auto-generated where possible
     if(IS_WASM)
         string(APPEND CMAKE_CXX_FLAGS " -sMEMORY64=2 -Wno-experimental -DSIMDJSON_NO_PORTABILITY_WARNING")
-        string(APPEND CMAKE_EXE_LINKER_FLAGS " -sINVOKE_RUN=0 -sALLOW_MEMORY_GROWTH=1 -sINITIAL_MEMORY=65536000 -sMEMORY_GROWTH_GEOMETRIC_STEP=0.50 -sMODULARIZE=1 -sEXPORT_NAME=AmalgamRuntime -sENVIRONMENT=worker,node -sEXPORTED_RUNTIME_METHODS=cwrap,ccall,FS,setValue,getValue,UTF8ToString -sEXPORTED_FUNCTIONS=_malloc,_free,_LoadEntity,_CloneEntity,_VerifyEntity,_StoreEntity,_ExecuteEntity,_ExecuteEntityJsonPtr,_DestroyEntity,_GetEntities,_SetRandomSeed,_SetJSONToLabel,_GetJSONPtrFromLabel,_SetSBFDataStoreEnabled,_IsSBFDataStoreEnabled,_GetVersionString,_SetMaxNumThreads,_GetMaxNumThreads,_GetConcurrencyTypeString,_DeleteString --preload-file /wasm/tzdata@/tzdata --preload-file /wasm/etc@/etc")
+        string(APPEND CMAKE_EXE_LINKER_FLAGS " -sINVOKE_RUN=0 -sALLOW_MEMORY_GROWTH=1 -sMEMORY_GROWTH_GEOMETRIC_STEP=0.50 -sMODULARIZE=1 -sEXPORT_NAME=AmalgamRuntime -sENVIRONMENT=worker,node,web -sEXPORTED_RUNTIME_METHODS=cwrap,ccall,FS,setValue,getValue,UTF8ToString -sEXPORTED_FUNCTIONS=_malloc,_free,_LoadEntity,_CloneEntity,_VerifyEntity,_StoreEntity,_ExecuteEntity,_ExecuteEntityJsonPtr,_DestroyEntity,_GetEntities,_SetRandomSeed,_SetJSONToLabel,_GetJSONPtrFromLabel,_SetSBFDataStoreEnabled,_IsSBFDataStoreEnabled,_GetVersionString,_SetMaxNumThreads,_GetMaxNumThreads,_GetConcurrencyTypeString,_DeleteString --preload-file /wasm/tzdata@/tzdata --preload-file /wasm/etc@/etc")
+        # Set memory arguments
+        if(CMAKE_BUILD_TYPE STREQUAL "Debug")
+            string(APPEND CMAKE_EXE_LINKER_FLAGS " -sINITIAL_HEAP=65536000 -sSTACK_SIZE=33554432")
+        else()
+            string(APPEND CMAKE_EXE_LINKER_FLAGS " -sINITIAL_HEAP=16777216 -sSTACK_SIZE=8388608")
+        endif()
     endif()
 
 elseif("${CMAKE_CXX_COMPILER_ID}" STREQUAL "AppleClang")
@@ -139,6 +145,15 @@ if(IS_MSVC)
     add_compile_definitions(UNICODE _UNICODE)
 endif()
 
+# Ensure that debug WASM builds include additional WASM-specific linker flags 
+if (IS_WASM AND CMAKE_BUILD_TYPE STREQUAL "Debug")
+    string(APPEND CMAKE_EXE_LINKER_FLAGS " -sASSERTIONS=2")
+    # Remove the below flags as they are incompatible with debugging WASM
+    string(REPLACE "-Werror" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
+    string(REPLACE "-Wlimited-postlink-optimizations" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
+    string(APPEND CMAKE_CXX_FLAGS " -O0 -gseparate-dwarf")
+endif()
+
 # amd64 advanced intrinsics:
 # Note: allowed values - avx avx2 avx512
 set(ADVANCED_INTRINSICS_AMD64 "avx2")

src/Amalgam/Parser.cpp

@@ -78,10 +78,10 @@ std::tuple<EvaluableNodeReference, std::vector<std::string>, size_t>
 		{
 			pt.originalSource = std::filesystem::canonical(p).string();
 		}
-		catch(std::filesystem::filesystem_error &e)
+		catch(...)
 		{
-			//file doesn't exist
-			pt.originalSource = e.what();
+			//file doesn't exist, or was some other form of resource, just use original
+			pt.originalSource = *original_source;
 		}
 	}
 

src/Amalgam/amlg_code/persist_module_test.amlg

@@ -1,5 +1,5 @@
-(parallel
-	#a 1
-	#hello
-	(print "hello\n")
-)
+(parallel
+	#a 1
+	#hello
+	(print "hello\n")
+)

src/Amalgam/amlg_code/persist_module_test/psm.amlg

@@ -1,5 +1,5 @@
-(parallel
-	#a 8
-	#hello
-	(print "hello from psm\n")
-)
+(parallel
+	#a 8
+	#hello
+	(print "hello from psm\n")
+)

src/Amalgam/amlg_code/persistent_tree_test_leaf.amlg

@@ -1,5 +1,5 @@
-(parallel
-	#f 6
-	#jello
-	(print "jello\n")
-)
+(parallel
+	#f 6
+	#jello
+	(print "jello\n")
+)

test/wasm_test/README.md

@@ -8,11 +8,11 @@ npm install
 
 ## Define environment variables
 
-Define the environment variable `AMALGAM_WASM_DIR` to the directory where the following files exist:
+Define the environment variable `AMALGAM_WASM_DIR` and `AMALGAM_BASE_FILE` to the directory where the following files exist:
 
-- amalgam-st.wasm
-- amalgam-st.cjs
-- amalgam-st.data
+- amalgam-st(-debug).wasm
+- amalgam-st(-debug).cjs
+- amalgam-st(-debug).data
 
 On your personal machine, you may find it easier to copy [.env.sample](./.env.sample) to `.env`.
 

test/wasm_test/tests/wasm.test.cjs

@@ -1,21 +1,30 @@
 const fs = require("node:fs");
 const path = require("node:path");
 const { describe, expect, test } = require("@jest/globals");
-const AmalgamRuntime = require(path.resolve(process.env.AMALGAM_WASM_DIR, "amalgam-st.cjs"));
+
+const AmalgamRuntime = require(path.resolve(
+  process.env.AMALGAM_WASM_DIR,
+  process.env.AMALGAM_BASE_FILE + ".cjs"
+));
 
 describe("Test Amalgam Webassembly", () => {
   let amlg;
 
   beforeAll(async () => {
-    const binary = fs.readFileSync(path.resolve(process.env.AMALGAM_WASM_DIR, "amalgam-st.wasm"));
+    const binary = fs.readFileSync(
+      path.resolve(
+        process.env.AMALGAM_WASM_DIR,
+        process.env.AMALGAM_BASE_FILE + ".wasm"
+      )
+    );
     amlg = await AmalgamRuntime({
-      "wasmBinary": binary,
-      "getPreloadedPackage": function (packagePath) {
+      wasmBinary: binary,
+      getPreloadedPackage: function (packagePath) {
         // Manually load package data from file system
         const data = fs.readFileSync(packagePath);
         return data.buffer;
       },
-      "locateFile": function (filepath) {
+      locateFile: function (filepath) {
         // Override the local file method to use local file system
         return path.resolve(process.env.AMALGAM_WASM_DIR, filepath);
       },