Loadable library recovery and example (KallistiOS#819)

* Fix and improve loadable library build.
* Fixed library ref counting and cleanup.
* Added library_lookup_fn.
* Aligned elf image to cache line and DMA.
* Added export_lookup_path and export_lookup_addr.
* Added loadable library full example.

.gitignore

@@ -1,5 +1,8 @@
 *.o
+*.a
 *.elf
+*.bin
+*.klf
 *.exe
 *.dll
 *.map
@@ -15,8 +18,6 @@ kernel/exports/kernel_exports.c
 kernel/arch/dreamcast/kernel/arch_exports.c
 kernel/arch/dreamcast/kernel/banner.h
 kernel/arch/dreamcast/sound/arm/stream.drv
-lib/dreamcast/*.a
-addons/lib/dreamcast/*.a
 utils/bincnv/bincnv
 utils/genromfs/genromfs
 utils/vqenc/vqenc
@@ -45,6 +46,10 @@ examples/dreamcast/pvr/bumpmap/romdisk/bumpmap.raw
 examples/dreamcast/pvr/modifier_volume_tex/romdisk/fruit.kmg
 examples/dreamcast/pvr/texture_render/texture_render.bin
 examples/dreamcast/video/screenshot/screenshot*.ppm
+examples/dreamcast/library/exports.c
+examples/dreamcast/library/exports_stubs.c
+examples/dreamcast/library/*/exports.c
+examples/dreamcast/library/*/exports_stubs.c
 utils/dc-chain/logs
 utils/dc-chain/*.tar.bz2
 utils/dc-chain/*.tar.gz

Makefile.rules

@@ -3,6 +3,7 @@
 # Makefile.rules
 # Copyright (c) 2000, 2001 Megan Potter
 # Copyright (c) 2024 Eric Fradella
+# Copyright (C) 2024 Ruslan Rostovtsev
 #
 
 # Global KallistiOS Makefile include
@@ -104,3 +105,20 @@ endef
 define KOS_GCCVER_MIN_WARNING
 @echo "Skipping $(TARGET) build as current GCC version ($(KOS_GCCVER)) is less than $(KOS_GCCVER_MIN)."
 endef
+
+ifdef EXPORTS_FILE
+
+OBJS += exports.o
+
+exports.o: exports.c
+exports.c: $(EXPORTS_FILE)
+	$(KOS_BASE)/utils/genexports/genexports.sh $(EXPORTS_FILE) exports.c $(EXPORTS_SYMBOL)
+
+exports_stubs.o: exports_stubs.c
+exports_stubs.c: $(EXPORTS_FILE)
+	$(KOS_BASE)/utils/genexports/genexportstubs.sh $(EXPORTS_FILE) exports_stubs.c
+
+$(TARGET_LIB): exports_stubs.o
+	$(KOS_AR) rcs $(TARGET_LIB) exports_stubs.o
+
+endif

environ_base.sh

@@ -47,6 +47,7 @@ export KOS_ADDR2LINE="${KOS_CC_BASE}/bin/${KOS_CC_PREFIX}-addr2line"
 export KOS_LD="${KOS_CC_BASE}/bin/${KOS_CC_PREFIX}-ld"
 export KOS_RANLIB="${KOS_CC_BASE}/bin/${KOS_CC_PREFIX}-gcc-ranlib"
 export KOS_STRIP="${KOS_CC_BASE}/bin/${KOS_CC_PREFIX}-strip"
+export KOS_SIZE="${KOS_CC_BASE}/bin/${KOS_CC_PREFIX}-size"
 export KOS_CFLAGS="${KOS_CFLAGS} ${KOS_INC_PATHS} -D_arch_${KOS_ARCH} -D_arch_sub_${KOS_SUBARCH} -Wall -g"
 export KOS_CPPFLAGS="${KOS_CPPFLAGS} ${KOS_INC_PATHS_CPP}"
 

examples/dreamcast/library/Makefile

@@ -0,0 +1,47 @@
+# KallistiOS ##version##
+#
+# examples/dreamcast/library/Makefile
+# Copyright (C) 2024 Ruslan Rostovtsev
+#
+
+TARGET_NAME = library-test
+TARGET = $(TARGET_NAME).elf
+TARGET_LIB = lib$(TARGET_NAME).a
+TARGET_BIN = $(TARGET_NAME).bin
+OBJS = $(TARGET_NAME).o romdisk.o
+
+EXPORTS_FILE = exports.txt
+EXPORTS_SYMBOL = libtest_symtab
+KOS_ROMDISK_DIR = romdisk
+
+include $(KOS_BASE)/Makefile.rules
+
+KOS_CFLAGS += -I./loadable-dependence
+
+all: rm-elf $(TARGET_LIB) loadable $(TARGET)
+
+clean: rm-elf
+	-rm -f $(OBJS)
+	-rm -rf ./romdisk
+	cd loadable-dependence && make clean
+	cd loadable-dependent && make clean
+
+rm-elf:
+	-rm -f $(TARGET) $(TARGET_BIN) $(TARGET_LIB) romdisk.*
+
+loadable:
+	mkdir -p romdisk
+	cd loadable-dependence && make && cp library-dependence.klf ../romdisk
+	cd loadable-dependent && make && cp library-dependent.klf ../romdisk
+
+$(TARGET): $(OBJS)
+	kos-cc -o $(TARGET) $(OBJS)
+
+run: $(TARGET)
+	$(KOS_LOADER) $(TARGET)
+
+dist: $(TARGET)
+	-rm -f $(OBJS) romdisk.img
+	$(KOS_STRIP) $(TARGET)
+	$(KOS_OBJCOPY) -R .stack -O binary $(TARGET) $(TARGET_BIN)
+	$(KOS_SIZE) $(TARGET)

examples/dreamcast/library/exports.txt

@@ -0,0 +1,7 @@
+include kos/exports.h
+include string.h
+
+# Additional newlib functions to export
+# Some part of it already exported by kernel/exports.txt
+strcmp
+strncmp

examples/dreamcast/library/library-test.c

@@ -0,0 +1,137 @@
+/* KallistiOS ##version##
+
+   library-test.c
+   Copyright (C) 2024 Ruslan Rostovtsev
+
+   This example program simply show how library works.
+*/
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdint.h>
+
+#include <dc/maple.h>
+#include <dc/maple/controller.h>
+
+#include <arch/arch.h>
+
+#include <kos/init.h>
+#include <kos/dbgio.h>
+#include <kos/dbglog.h>
+#include <kos/library.h>
+#include <kos/exports.h>
+
+#include "library-dependence.h"
+
+KOS_INIT_FLAGS(INIT_DEFAULT | INIT_EXPORT);
+
+extern export_sym_t libtest_symtab[];
+static symtab_handler_t st_libtest = {
+    {
+        "sym/library/test",
+        0,
+        0x00010000,
+        0,
+        NMMGR_TYPE_SYMTAB,
+        NMMGR_LIST_INIT
+    },
+    libtest_symtab
+};
+
+static void __attribute__((__noreturn__)) wait_exit(void) {
+    maple_device_t *dev;
+    cont_state_t *state;
+
+    dbglog(DBG_DEBUG, "Press any button to exit.\n");
+
+    for(;;) {
+        dev = maple_enum_type(0, MAPLE_FUNC_CONTROLLER);
+
+        if(dev) {
+            state = (cont_state_t *)maple_dev_status(dev);
+            if(state) {
+                if(state->buttons) {
+                    arch_exit();
+                }
+            }
+        }
+    }
+}
+
+int main(int argc, char *argv[]) {
+
+    klibrary_t *lib_dependence;
+    klibrary_t *lib_dependent;
+    export_sym_t *sym;
+    uint32_t ver;
+    library_test_func_t library_test_func;
+    library_test_func2_t library_test_func2;
+
+    // dbgio_dev_select("fb");
+    dbglog(DBG_DEBUG, "Initializing exports.\n");
+
+    if(nmmgr_handler_add(&st_libtest.nmmgr) < 0) {
+        dbglog(DBG_ERROR, "Failed.");
+        wait_exit();                             
+        return -1;
+    }
+
+    dbglog(DBG_DEBUG, "Loading /rd/library-dependence.klf\n");
+    lib_dependence = library_open("dependence", "/rd/library-dependence.klf");
+
+    if (lib_dependence == NULL) {
+        dbglog(DBG_ERROR, "Loading failed.\n");
+        wait_exit();
+        return -1;
+    }
+
+    ver = library_get_version(lib_dependence);
+
+    dbglog(DBG_DEBUG, "Successfully loaded: %s v%ld.%ld.%ld\n",
+        library_get_name(lib_dependence),
+        (ver >> 16) & 0xff, (ver >> 8) & 0xff, ver & 0xff);
+
+    dbglog(DBG_DEBUG, "Loading /rd/library-dependence.klf\n");
+    lib_dependent = library_open("dependent", "/rd/library-dependent.klf");
+
+    if (lib_dependence == NULL) {
+        dbglog(DBG_ERROR, "Loading failed.\n");
+        wait_exit();
+        return -1;
+    }
+
+    ver = library_get_version(lib_dependent);
+
+    dbglog(DBG_DEBUG, "Successfully loaded: %s v%ld.%ld.%ld\n",
+        library_get_name(lib_dependent),
+        (ver >> 16) & 0xff, (ver >> 8) & 0xff, ver & 0xff);
+
+    dbglog(DBG_DEBUG, "Testing exports runtime on host\n");
+
+    sym = export_lookup("library_test_func");
+
+    if (sym && sym->ptr != (uint32_t)-1) {
+        library_test_func = (library_test_func_t)sym->ptr;
+        library_test_func(444);
+    }
+    else {
+        dbglog(DBG_ERROR, "Lookup symbol failed: library_test_func");
+    }
+
+    sym = export_lookup("library_test_func2");
+
+    if (sym && sym->ptr != (uint32_t)-1) {
+        library_test_func2 = (library_test_func2_t)sym->ptr;
+        library_test_func2("Hello from library test");
+    }
+    else {
+        dbglog(DBG_ERROR, "Lookup symbol failed: library_test_func");
+    }
+
+    library_close(lib_dependent);
+    library_close(lib_dependence);
+    nmmgr_handler_remove(&st_libtest.nmmgr);
+
+    wait_exit();
+    return 0;
+}

examples/dreamcast/library/loadable-dependence/Makefile

@@ -0,0 +1,24 @@
+# KallistiOS ##version##
+#
+# examples/dreamcast/library/loadable-dependence/Makefile
+# Copyright (C) 2024 Ruslan Rostovtsev
+#
+
+TARGET_NAME = library-dependence
+TARGET = $(TARGET_NAME).klf
+TARGET_LIB = lib$(TARGET_NAME).a
+OBJS = $(TARGET_NAME).o
+
+# For exporting kos_md5
+LIBS = -lkosutils
+
+# library-test exported stub for link test
+DBG_LIBS = -llibrary-test
+
+EXPORTS_SYMBOL = library_symtab
+EXPORTS_FILE = exports.txt
+
+KOS_CFLAGS += -I./
+KOS_LIB_PATHS += -L../
+
+include $(KOS_BASE)/loadable/Makefile.prefab

examples/dreamcast/library/loadable-dependence/exports.txt

@@ -0,0 +1,10 @@
+include kos/exports.h
+include kos/md5.h
+include library-dependence.h
+
+# Loadable library exported functions
+library_test_func
+library_test_func2
+
+# Linked library exported functions
+kos_md5

examples/dreamcast/library/loadable-dependence/library-dependence.c

@@ -0,0 +1,55 @@
+/* KallistiOS ##version##
+
+   library-dependence.c
+   Copyright (C) 2024 Ruslan Rostovtsev
+
+   This example program simply show how library works.
+*/
+
+#include <stdint.h>
+#include <kos/dbglog.h>
+#include <kos/library.h>
+#include <kos/exports.h>
+#include <kos/version.h>
+
+extern export_sym_t library_symtab[];
+static symtab_handler_t library_hnd = {
+    {
+        "sym/library/dependence",
+        0,
+        0x00010000,
+        0,
+        NMMGR_TYPE_SYMTAB,
+        NMMGR_LIST_INIT
+    },
+    library_symtab
+};
+
+/* Library functions */
+const char *lib_get_name() {
+    return library_hnd.nmmgr.pathname + 12;
+}
+
+uint32_t lib_get_version() {
+    return KOS_VERSION_MAKE(1, 0, 0);
+}
+
+int lib_open(klibrary_t *lib) {
+    dbglog(DBG_DEBUG, "Library \"%s\" opened.\n", lib_get_name());
+    return nmmgr_handler_add(&library_hnd.nmmgr);
+}
+
+int lib_close(klibrary_t *lib) {
+    dbglog(DBG_DEBUG, "Library \"%s\" closed.\n", lib_get_name());
+    return nmmgr_handler_remove(&library_hnd.nmmgr);
+}
+
+/* Exported functions */
+int library_test_func(int arg) {
+    dbglog(DBG_DEBUG, "Library \"%s\" test int: %d\n", lib_get_name(), arg);
+    return 0;
+}
+
+void library_test_func2(const char *arg) {
+    dbglog(DBG_DEBUG, "Library \"%s\" test char: %s\n", lib_get_name(), arg);
+}

examples/dreamcast/library/loadable-dependence/library-dependence.h

@@ -0,0 +1,18 @@
+/* KallistiOS ##version##
+
+   library-dependence.h
+   Copyright (C) 2024 Ruslan Rostovtsev
+
+   This example program simply show how library works.
+*/
+
+#include <stdint.h>
+
+/**
+ * @brief Exported test functions
+ */
+int library_test_func(int arg);
+void library_test_func2(const char *arg);
+
+typedef int (*library_test_func_t)(int arg);
+typedef void (*library_test_func2_t)(const char *arg);

examples/dreamcast/library/loadable-dependent/Makefile

@@ -0,0 +1,18 @@
+# KallistiOS ##version##
+#
+# examples/dreamcast/library/loadable-dependent/Makefile
+# Copyright (C) 2024 Ruslan Rostovtsev
+#
+
+TARGET_NAME = library-dependent
+TARGET = $(TARGET_NAME).klf
+OBJS = $(TARGET_NAME).o
+
+# Dependence library and host stubs for link test
+DBG_LIBS = -llibrary-dependence -llibrary-test
+KOS_LIB_PATHS += -L../loadable-dependence -L../
+
+# Dependence include
+KOS_CFLAGS += -I../loadable-dependence
+
+include $(KOS_BASE)/loadable/Makefile.prefab