Skip to content

Commit

Permalink
rework compiler wrapper sh->py
Browse files Browse the repository at this point in the history
  • Loading branch information
pmp-p committed Oct 12, 2024
1 parent dca09dc commit d3bb6b1
Show file tree
Hide file tree
Showing 10 changed files with 424 additions and 390 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ jobs:
build:
runs-on: ubuntu-22.04
env:
BUILDS: 3.13 3.14
BUILDS: 3.12 3.13 3.14
EMFLAVOUR: tot

steps:
Expand Down
10 changes: 2 additions & 8 deletions config
Original file line number Diff line number Diff line change
Expand Up @@ -71,15 +71,9 @@ fi
EXTRABINS="/usr/local/bin /opt/bin"

# ============ emscripten ==============

# stable
# export EMFLAVOUR=${EMFLAVOUR:3.1.65}

# embind broke in 3.1.48-tot (coro suspend/resume)
# embding more broken in 3.1.51

# stable==latest dev==tot
export EMFLAVOUR=${EMFLAVOUR:latest}

export EMSDK_QUIET=1


#temp fix for oom on CI (Error: Process completed with exit code 143.)
Expand Down
248 changes: 248 additions & 0 deletions emsdk-cc
Original file line number Diff line number Diff line change
@@ -0,0 +1,248 @@
#!/usr/bin/env python3

import sys
import os
from pathlib import Path


def dbg(*argv, **kw):
kw.setdefault("file", sys.stderr)
return print(*argv, **kw)


sys.argv.pop(0)

EXEC = sys.argv.pop(0)

args = sys.argv


def env(k, default):
if default is false:
default = "false"
if default is true:
default = "true"

v = os.environ.get(k, default)
if v == "false":
return False
if v == "true":
return True
return v.strip()


def arglist(*argv):
al = " ".join(argv)
al = al.replace("\n", " ")
while al.find(" ") >= 0:
al = al.replace(" ", " ")
return al.strip().split(" ")


# -Wwarn-absolute-paths
# --valid-abspath ${SDKROOT}
# COMMON="-Wno-unsupported-floating-point-opt"

COMMON = arglist(
"""
-Wno-limited-postlink-optimizations
-Wno-unused-command-line-argument
-Wno-unreachable-code-fallthrough
-Wno-unused-function
""",
os.environ.get("PYDK_CFLAGS", ""),
)

false = False
true = True

PY_MODULE = IS_SHARED = false
SHARED_TARGET = SHARED = ""

MVP = env("MVP", true)

if MVP:
# turn of wasm ex (https://github.com/emscripten-core/emscripten/pull/20536)
# -fno-wasm-exceptions -sEMSCRIPTEN_LONGJMP=0

# -mcpu=generic would activate those https://reviews.llvm.org/D125728
# https://github.com/emscripten-core/emscripten/pull/17689

# -fPIC not allowed with -mno-mutable-globals
# -mno-sign-ext not allowed with pthread

# WASMOPTS="-fno-wasm-exceptions -sSUPPORT_LONGJMP=emscripten"
# CPU="-mnontrapping-fptoint -mno-reference-types -mno-sign-ext -m32"

CPU = arglist(
"""
-D_FILE_OFFSET_BITS=64
-sSUPPORT_LONGJMP=emscripten
-mno-bulk-memory
-mnontrapping-fptoint
-mno-reference-types
-mno-sign-ext
-m32
"""
)
else:
CPU = arglist("-D_FILE_OFFSET_BITS=64 -mcpu=bleeding-edge -m64")


# try to keep 32 but maybe with 64 iface (bigint)
WASM_EXTRA = env("WASM_EXTRA", "") + " " + env("WASM_OPTS", "")

COPTS = env("COPTS", "-fPIC")
MAIN_MODULE = LINKING = False
EXE = ""
MODE = ""

SKIP = False
COMPILE = False

out = []
for argc, arg in enumerate(sys.argv):
if arg in ("-v", "--version"):
SKIP = True
break

if arg.startswith('CMakeFiles/'):
SKIP = True
break


if arg.find("MAIN_MODULE"):
MAIN_MODULE = True

if arg.lower() in ("-fpic", "-latomic"):
continue

if arg in ("-Wl,--as-needed", "-Wl,--eh-frame-hdr", "-Wl,-znoexecstack", "-Wl,-znow", "-Wl,-zrelro", "-Wl,-zrelro,-znow"):
continue

if arg in ("-O3", "-g", "-lgcc", "-lgcc_s", "-fallow-argument-mismatch"):
continue

if arg == "-pthread":
if MVP:
continue

if arg in ("-o", "-c"):
CPU_EXTRA = WASM_EXTRA
MODE = arg
MODE_POS = argc
if arg == "-c":
COMPILE = True
# TODO maybe add node runner and compile to .cjs
elif arg == "-o":
EXE_POS = argc + 1
EXE = sys.argv[EXE_POS]

elif arg.endswith(".so") or arg == "-shared":
LINKING = True
if arg == "-shared":
IS_SHARED = True
elif arg.find("wasm32-emscripten.so") > 0 or arg.find("abi3.so") > 0:
IS_SHARED = True
PY_MODULE = true
SHARED_TARGET = arg

if IS_SHARED:
out.append(arg)
SHARED = "-sSIDE_MODULE"
if not SHARED in out:
out.insert(0, SHARED)
out.append(f"-L{os.environ['PREFIX']}/lib")
continue

# prevent duplicates objects/archives files on cmdline when linking
if LINKING or MODE=="-o":
if arg.endswith(".a") or arg.endswith(".o") or arg.startswith("-l"):
if arg in out:
continue

# that is for some very bad setup.py behaviour regarding cross compiling.
# should not be needed ..
if arg.startswith("-I/usr/"):
continue

if arg.startswith("-L/usr/"):
continue

out.append(arg)

os.environ.pop("_EMCC_CCACHE", "")

"""
if [ "\$arg" = "-lutil" ]
then
continue
fi
if [ "\$arg" = "-nomvp" ]
then
MVP=false
continue
fi
if \$MVP
then
if \$WASM_PURE
then
SOTMP=\$(mktemp).so
mv \$SHARED_TARGET \$SOTMP
# --memory64-lowering --signext-lowering
$SDKROOT/emsdk/upstream/bin/wasm-emscripten-finalize -mvp \$SOTMP -o \$SHARED_TARGET
[ -f \$SHARED_TARGET.map ] && rm \$SHARED_TARGET.map
rm \$SOTMP
fi
fi
"""

final = [EXEC]
if SKIP:
final.extend(sys.argv)
else:
if IS_SHARED:
final.extend(arglist(SHARED, COPTS))
final.extend(CPU)
final.extend(arglist(WASM_EXTRA, env("LDFLAGS", ""), "-gsource-map --source-map-base /"))

# do not pass WASM opts when -c/-o but always PIC
elif MAIN_MODULE:
final.extend(arglist(SHARED, COPTS))
final.extend(CPU)
final.extend(arglist(env("CPU_EXTRA", ""), env("CPPFLAGS", ""), "-DBUILD_STATIC -gsource-map --source-map-base /"))
else:
final.extend(arglist(SHARED, COPTS, env("CPU_EXTRA", ""), env("CPPFLAGS", ""), "-DBUILD_STATIC"))
final.extend(out)
final.extend(COMMON)

if env("EMCC_TRACE", false):
dbg(
f"""
{COMMON=}
{CPU=}
{out=}
{LINKING=}
{PY_MODULE=} {SHARED_TARGET=}
{MODE=} {EXE=}
{final=}
""")

sys.path.insert(0, str(Path(EXEC).parent))
sys.argv.clear()
while len(final):
arg = final.pop(0)
#arg = arg.replace('"', '\\"')
sys.argv.append(arg)
exec(open(EXEC, "r").read(), globals(), globals())
17 changes: 9 additions & 8 deletions python-wasi-sdk.sh
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ do
echo "doing a clean build"
if [ -d ${SDKROOT}/go ]
then
chown -R u+wxr ${SDKROOT}/build ${SDKROOT}/go
chown -R u+rwx ${SDKROOT}/build ${SDKROOT}/go
fi
rm -rf ${SDKROOT}/* ${SDKROOT}/.??*
fi
Expand Down Expand Up @@ -190,14 +190,17 @@ END

if [ -f /tmp/sdk/emsdk.tar ]
then
echo " using cache "
echo " using cached cpython-build-emsdk-deps"
else
if ./scripts/cpython-build-emsdk-deps.sh
then
if [ -f /pp ]
if $CI
then
pushd /
tar -cpR $SDKROOT \
--exclude=${SDKROOT}/devices/*/usr/bin/*3.1* \
--exclude=${SDKROOT}/devices/*/usr/lib/python3.1? \
--exclude=${SDKROOT}/devices/*/usr/include/python3.1? \
--exclude=${SDKROOT}/config \
--exclude=${SDKROOT}/*sh \
--exclude=${SDKROOT}/scripts/* \
Expand All @@ -210,18 +213,16 @@ END
fi
else
echo " cpython-build-emsdk-deps failed" 1>&2
exit 182
exit 213
fi
fi



echo " ------------ building cpython wasm ${PYBUILD} ${CIVER} ----------------" 1>&2
if ./scripts/cpython-build-emsdk.sh > /dev/null
then

echo " --------- adding some usefull pkg ${PYBUILD} ${CIVER} ---------" 1>&2
./scripts/cpython-build-emsdk-prebuilt.sh || exit 213
./scripts/cpython-build-emsdk-prebuilt.sh || exit 223

echo "
Expand All @@ -237,7 +238,7 @@ END

else
echo " cpython-build-emsdk failed" 1>&2
exit 207
exit 239
fi

fi
Expand Down
17 changes: 9 additions & 8 deletions python-wasm-sdk.sh
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ do
echo "doing a clean build"
if [ -d ${SDKROOT}/go ]
then
chown -R u+wxr ${SDKROOT}/build ${SDKROOT}/go
chown -R u+rwx ${SDKROOT}/build ${SDKROOT}/go
fi
rm -rf ${SDKROOT}/* ${SDKROOT}/.??*
fi
Expand Down Expand Up @@ -190,14 +190,17 @@ END

if [ -f /tmp/sdk/emsdk.tar ]
then
echo " using cache "
echo " using cached cpython-build-emsdk-deps"
else
if ./scripts/cpython-build-emsdk-deps.sh
then
if [ -f /pp ]
if $CI
then
pushd /
tar -cpR $SDKROOT \
--exclude=${SDKROOT}/devices/*/usr/bin/*3.1* \
--exclude=${SDKROOT}/devices/*/usr/lib/python3.1? \
--exclude=${SDKROOT}/devices/*/usr/include/python3.1? \
--exclude=${SDKROOT}/config \
--exclude=${SDKROOT}/*sh \
--exclude=${SDKROOT}/scripts/* \
Expand All @@ -210,18 +213,16 @@ END
fi
else
echo " cpython-build-emsdk-deps failed" 1>&2
exit 182
exit 213
fi
fi



echo " ------------ building cpython wasm ${PYBUILD} ${CIVER} ----------------" 1>&2
if ./scripts/cpython-build-emsdk.sh > /dev/null
then

echo " --------- adding some usefull pkg ${PYBUILD} ${CIVER} ---------" 1>&2
./scripts/cpython-build-emsdk-prebuilt.sh || exit 213
./scripts/cpython-build-emsdk-prebuilt.sh || exit 223

echo "
Expand All @@ -237,7 +238,7 @@ END

else
echo " cpython-build-emsdk failed" 1>&2
exit 207
exit 239
fi

fi
Expand Down
Loading

0 comments on commit d3bb6b1

Please sign in to comment.