net: enable native golang linux networking

GNUmakefile

@@ -926,6 +926,7 @@ endif
 	@cp -rp lib/musl/src/env             build/release/tinygo/lib/musl/src
 	@cp -rp lib/musl/src/errno           build/release/tinygo/lib/musl/src
 	@cp -rp lib/musl/src/exit            build/release/tinygo/lib/musl/src
+	@cp -rp lib/musl/src/fcntl           build/release/tinygo/lib/musl/src
 	@cp -rp lib/musl/src/include         build/release/tinygo/lib/musl/src
 	@cp -rp lib/musl/src/internal        build/release/tinygo/lib/musl/src
 	@cp -rp lib/musl/src/legacy          build/release/tinygo/lib/musl/src

builder/musl.go

@@ -116,6 +116,7 @@ var libMusl = Library{
 			"env/*.c",
 			"errno/*.c",
 			"exit/*.c",
+			"fcntl/*.c",
 			"internal/defsysinfo.c",
 			"internal/libc.c",
 			"internal/syscall_ret.c",

loader/goroot.go

@@ -45,7 +45,7 @@ func GetCachedGoroot(config *compileopts.Config) (string, error) {
 	}
 
 	// Find the overrides needed for the goroot.
-	overrides := pathsToOverride(config.GoMinorVersion, needsSyscallPackage(config.BuildTags()))
+	overrides := pathsToOverride(config.GoMinorVersion, config.BuildTags())
 
 	// Resolve the merge links within the goroot.
 	merge, err := listGorootMergeLinks(goroot, tinygoroot, overrides)
@@ -225,9 +225,28 @@ func needsSyscallPackage(buildTags []string) bool {
 	return false
 }
 
-// The boolean indicates whether to merge the subdirs. True means merge, false
-// means use the TinyGo version.
-func pathsToOverride(goMinor int, needsSyscallPackage bool) map[string]bool {
+// linuxNetworking returns whether the unmodified go linux net stack should be used
+// until the full rework of the net package is done.
+// To ensure the correct build target, check for the following tags:
+// linux && !baremetal && !nintendoswitch && !tinygo.wasm
+func linuxNetworking(buildTags []string) bool {
+	targetLinux := false
+	for _, tag := range buildTags {
+		if tag == "linux" {
+			targetLinux = true
+		}
+		if tag == "baremetal" || tag == "nintendoswitch" || tag == "tinygo.wasm" {
+			return false
+		}
+	}
+	return targetLinux
+}
+
+// The boolean indicates whether to merge the subdirs.
+//
+// True: Merge the golang and tinygo source directories.
+// False: Uses the TinyGo version exclusively.
+func pathsToOverride(goMinor int, buildTags []string) map[string]bool {
 	paths := map[string]bool{
 		"":                            true,
 		"crypto/":                     true,
@@ -249,7 +268,7 @@ func pathsToOverride(goMinor int, needsSyscallPackage bool) map[string]bool {
 		"internal/task/":              false,
 		"internal/wasi/":              false,
 		"machine/":                    false,
-		"net/":                        true,
+		"net/":                        true, // this is important if the GOOS != linux
 		"net/http/":                   false,
 		"os/":                         true,
 		"reflect/":                    false,
@@ -265,9 +284,18 @@ func pathsToOverride(goMinor int, needsSyscallPackage bool) map[string]bool {
 		paths["crypto/internal/boring/sig/"] = false
 	}
 
-	if needsSyscallPackage {
+	if needsSyscallPackage(buildTags) {
 		paths["syscall/"] = true // include syscall/js
 	}
+
+	// To make sure the correct version of the net package is used, it is advised
+	// to clean the go cache before building
+	if linuxNetworking(buildTags) {
+		for _, v := range []string{"crypto/tls/", "net/http/", "net/"} {
+			delete(paths, v) // remote entries so go stdlib is used
+		}
+	}
+
 	return paths
 }
 

loader/loader.go

@@ -279,7 +279,7 @@ func (p *Program) getOriginalPath(path string) string {
 			originalPath = realgorootPath
 		}
 		maybeInTinyGoRoot := false
-		for prefix := range pathsToOverride(p.config.GoMinorVersion, needsSyscallPackage(p.config.BuildTags())) {
+		for prefix := range pathsToOverride(p.config.GoMinorVersion, p.config.BuildTags()) {
 			if runtime.GOOS == "windows" {
 				prefix = strings.ReplaceAll(prefix, "/", "\\")
 			}

main_test.go

@@ -341,8 +341,7 @@ func emuCheck(t *testing.T, options compileopts.Options) {
 	}
 	if spec.Emulator != "" {
 		emulatorCommand := strings.SplitN(spec.Emulator, " ", 2)[0]
-		_, err := exec.LookPath(emulatorCommand)
-		if err != nil {
+		if _, err := exec.LookPath(emulatorCommand); err != nil {
 			if errors.Is(err, exec.ErrNotFound) {
 				t.Skipf("emulator not installed: %q", emulatorCommand)
 			}
@@ -672,8 +671,7 @@ func TestWasmExport(t *testing.T) {
 			builder := r.NewHostModuleBuilder("tester")
 			builder.NewFunctionBuilder().WithFunc(callOutside).Export("callOutside")
 			builder.NewFunctionBuilder().WithFunc(callTestMain).Export("callTestMain")
-			_, err = builder.Instantiate(ctx)
-			if err != nil {
+			if _, err = builder.Instantiate(ctx); err != nil {
 				t.Fatal(err)
 			}
 
@@ -730,8 +728,7 @@ func TestWasmFuncOf(t *testing.T) {
 	cmd := exec.Command("node", "testdata/wasmfunc.js", result.Binary, buildConfig.BuildMode())
 	cmd.Stdout = output
 	cmd.Stderr = output
-	err = cmd.Run()
-	if err != nil {
+	if err = cmd.Run(); err != nil {
 		t.Error("failed to run node:", err)
 	}
 	checkOutput(t, "testdata/wasmfunc.txt", output.Bytes())
@@ -771,8 +768,8 @@ func TestWasmExportJS(t *testing.T) {
 			cmd := exec.Command("node", "testdata/wasmexport.js", result.Binary, buildConfig.BuildMode())
 			cmd.Stdout = output
 			cmd.Stderr = output
-			err = cmd.Run()
-			if err != nil {
+
+			if err = cmd.Run(); err != nil {
 				t.Error("failed to run node:", err)
 			}
 			checkOutput(t, "testdata/wasmexport.txt", output.Bytes())
@@ -1037,8 +1034,7 @@ func TestMain(m *testing.M) {
 		switch os.Args[1] {
 		case "clang", "ld.lld", "wasm-ld":
 			// Invoke a specific tool.
-			err := builder.RunTool(os.Args[1], os.Args[2:]...)
-			if err != nil {
+			if err := builder.RunTool(os.Args[1], os.Args[2:]...); err != nil {
 				// The tool should have printed an error message already.
 				// Don't print another error message here.
 				os.Exit(1)

src/runtime/netpoll.go

@@ -0,0 +1,47 @@
+// Copyright 2024 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build linux
+
+package runtime
+
+// For debugging purposes this is used for all target architectures, but this is only valid for linux systems.
+// TODO: add linux specific build tags
+type pollDesc struct {
+	runtimeCtx uintptr
+}
+
+func (pd *pollDesc) wait(mode int, isFile bool) error {
+	return nil
+}
+
+const (
+	pollNoError        = 0 // no error
+	pollErrClosing     = 1 // descriptor is closed
+	pollErrTimeout     = 2 // I/O timeout
+	pollErrNotPollable = 3 // general error polling descriptor
+)
+
+//go:linkname poll_runtime_pollReset internal/poll.runtime_pollReset
+func poll_runtime_pollReset(pd *pollDesc, mode int) int {
+	// println("poll_runtime_pollReset not implemented", pd, mode)
+	return pollNoError
+}
+
+//go:linkname poll_runtime_pollWait internal/poll.runtime_pollWait
+func poll_runtime_pollWait(pd *pollDesc, mode int) int {
+	// println("poll_runtime_pollWait not implemented", pd, mode)
+	return pollNoError
+}
+
+//go:linkname poll_runtime_pollSetDeadline internal/poll.runtime_pollSetDeadline
+func poll_runtime_pollSetDeadline(pd *pollDesc, d int64, mode int) {
+	// println("poll_runtime_pollSetDeadline not implemented", pd, d, mode)
+}
+
+//go:linkname poll_runtime_pollOpen internal/poll.runtime_pollOpen
+func poll_runtime_pollOpen(fd uintptr) (*pollDesc, int) {
+	// println("poll_runtime_pollOpen not implemented", fd)
+	return &pollDesc{runtimeCtx: uintptr(0x1337)}, pollNoError
+}

src/runtime/netpoll_generic.go

@@ -0,0 +1,38 @@
+// Copyright 2024 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build !linux
+
+package runtime
+
+const (
+	pollNoError        = 0 // no error
+	pollErrClosing     = 1 // descriptor is closed
+	pollErrTimeout     = 2 // I/O timeout
+	pollErrNotPollable = 3 // general error polling descriptor
+)
+
+// Network poller descriptor.
+//
+// No heap pointers.
+// For linux to call create Fds with a pollDesc, it needs a ctxRuntime pointer, so use the original pollDesc struct.
+// On linux we have a heap.
+type pollDesc struct{}
+
+//go:linkname poll_runtime_pollReset internal/poll.runtime_pollReset
+func poll_runtime_pollReset(pd *pollDesc, mode int) int {
+	println("poll_runtime_pollReset not implemented", pd, mode)
+	return pollErrClosing
+}
+
+//go:linkname poll_runtime_pollWait internal/poll.runtime_pollWait
+func poll_runtime_pollWait(pd *pollDesc, mode int) int {
+	println("poll_runtime_pollWait not implemented", pd, mode)
+	return pollErrClosing
+}
+
+//go:linkname poll_runtime_pollSetDeadline internal/poll.runtime_pollSetDeadline
+func poll_runtime_pollSetDeadline(pd *pollDesc, d int64, mode int) {
+	println("poll_runtime_pollSetDeadline not implemented", pd, d, mode)
+}

src/runtime/poll.go

@@ -4,20 +4,21 @@ package runtime
 
 //go:linkname poll_runtime_pollServerInit internal/poll.runtime_pollServerInit
 func poll_runtime_pollServerInit() {
-	panic("todo: runtime_pollServerInit")
+	// fmt.Printf("poll_runtime_pollServerInit not implemented, skipping panic\n")
 }
 
-//go:linkname poll_runtime_pollOpen internal/poll.runtime_pollOpen
-func poll_runtime_pollOpen(fd uintptr) (uintptr, int) {
-	panic("todo: runtime_pollOpen")
-}
+// //go:linkname poll_runtime_pollOpen internal/poll.runtime_pollOpen
+// func poll_runtime_pollOpen(fd uintptr) (uintptr, int) {
+// 	// fmt.Printf("poll_runtime_pollOpen not implemented, skipping panic\n")
+// 	return 0, 0
+// }
 
 //go:linkname poll_runtime_pollClose internal/poll.runtime_pollClose
 func poll_runtime_pollClose(ctx uintptr) {
-	panic("todo: runtime_pollClose")
+	// fmt.Printf("poll_runtime_pollClose not implemented, skipping panic\n")
 }
 
 //go:linkname poll_runtime_pollUnblock internal/poll.runtime_pollUnblock
 func poll_runtime_pollUnblock(ctx uintptr) {
-	panic("todo: runtime_pollUnblock")
+	// fmt.Printf("poll_runtime_pollUnblock not implemented, skipping panic\n")
 }

src/runtime/sync.go

@@ -1,13 +1,72 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
 package runtime
 
+import (
+	"sync/atomic"
+	"unsafe"
+)
+
 // This file contains stub implementations for internal/poll.
+// The official golang implementation states:
+//
+// "That is, don't think of these as semaphores.
+// Think of them as a way to implement sleep and wakeup
+// such that every sleep is paired with a single wakeup,
+// even if, due to races, the wakeup happens before the sleep."
+//
+// This is an experimental and probably incomplete implementation of the
+// semaphore system, tailed to the network use case. That means, that it does not
+// implement the modularity that the semacquire/semacquire1 implementation model
+// offers, which in fact is emitted here entirely.
+// This means we assume the following constant settings from the golang standard
+// library: lifo=false,profile=semaBlock,skipframe=0,reason=waitReasonSemaquire
+
+type semaRoot struct {
+	nwait atomic.Uint32
+}
+
+var semtable semTable
+
+// Prime to not correlate with any user patterns.
+const semTabSize = 251
+
+type semTable [semTabSize]struct {
+	root semaRoot
+	pad  [64 - unsafe.Sizeof(semaRoot{})]byte // only 64 x86_64, make this variable
+}
+
+func (t *semTable) rootFor(addr *uint32) *semaRoot {
+	return &t[(uintptr(unsafe.Pointer(addr))>>3)%semTabSize].root
+}
 
 //go:linkname semacquire internal/poll.runtime_Semacquire
 func semacquire(sema *uint32) {
-	panic("todo: semacquire")
+	if cansemacquire(sema) {
+		return
+	}
+}
+
+// Copied from src/runtime/sema.go
+func cansemacquire(addr *uint32) bool {
+	for {
+		v := atomic.LoadUint32(addr)
+		if v == 0 {
+			return false
+		}
+		if atomic.CompareAndSwapUint32(addr, v, v-1) {
+			return true
+		}
+	}
 }
 
 //go:linkname semrelease internal/poll.runtime_Semrelease
 func semrelease(sema *uint32) {
-	panic("todo: semrelease")
+	root := semtable.rootFor(sema)
+	atomic.AddUint32(sema, 1)
+	if root.nwait.Load() == 0 {
+		return
+	}
 }

src/syscall/forklock.go

@@ -0,0 +1,17 @@
+//go:build tinygo && linux && !wasip1 && !wasip2 && tinygo.wasm && !wasm_unknown && !darwin && !baremetal && !nintendoswitch
+
+package syscall
+
+import (
+	"sync"
+)
+
+var ForkLock sync.RWMutex
+
+func CloseOnExec(fd int) {
+	system.CloseOnExec(fd)
+}
+
+func SetNonblock(fd int, nonblocking bool) (err error) {
+	return system.SetNonblock(fd, nonblocking)
+}

src/syscall/syscall_libc.go

@@ -215,7 +215,11 @@ func Truncate(path string, length int64) (err error) {
 func Faccessat(dirfd int, path string, mode uint32, flags int) (err error)
 
 func Kill(pid int, sig Signal) (err error) {
-	return ENOSYS // TODO
+	result := libc_kill(int32(pid), int32(sig))
+	if result < 0 {
+		err = getErrno()
+	}
+	return
 }
 
 type SysProcAttr struct{}
@@ -470,3 +474,8 @@ func libc_execve(filename *byte, argv **byte, envp **byte) int
 //
 //export truncate
 func libc_truncate(path *byte, length int64) int32
+
+// int kill(pid_t pid, int sig);
+//
+//export kill
+func libc_kill(pid, sig int32) int32