8338123: Linker crash when building a downcall handle with many argum…

…ents in x64

Reviewed-by: mcimadamore

src/hotspot/cpu/x86/downcallLinker_x86_64.cpp

@@ -34,8 +34,8 @@
 
 #define __ _masm->
 
-static const int native_invoker_code_base_size = 512;
-static const int native_invoker_size_per_arg = 8;
+static const int native_invoker_code_base_size = 256;
+static const int native_invoker_size_per_arg = 16;
 
 RuntimeStub* DowncallLinker::make_downcall_stub(BasicType* signature,
                                                 int num_args,

test/jdk/java/foreign/largestub/TestLargeStub.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2023, 2024, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -25,29 +25,39 @@
  * @test
  * @library ../
  * @modules java.base/jdk.internal.foreign
- * @run testng/othervm --enable-native-access=ALL-UNNAMED TestLargeStub
+ * @run junit/othervm --enable-native-access=ALL-UNNAMED TestLargeStub
  */
 
-import org.testng.annotations.Test;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
 
 import java.lang.foreign.FunctionDescriptor;
 import java.lang.foreign.Linker;
 import java.lang.foreign.MemoryLayout;
+import java.lang.foreign.ValueLayout;
 import java.util.stream.Stream;
 
+import static org.junit.jupiter.params.provider.Arguments.arguments;
+
 public class TestLargeStub extends NativeTestHelper {
 
+    private static final int DOWNCALL_AVAILABLE_SLOTS = 248;
+    private static final int UPCALL_AVAILABLE_SLOTS = 250;
+
     MemoryLayout STRUCT_LL = MemoryLayout.structLayout(
         C_LONG_LONG,
         C_LONG_LONG
     ); // 16 byte struct triggers return buffer usage on SysV
 
-    @Test
-    public void testDowncall() {
+    @ParameterizedTest
+    @MethodSource("layouts")
+    public void testDowncall(ValueLayout layout, int numSlots) {
         // Link a handle with a large number of arguments, to try and overflow the code buffer
         Linker.nativeLinker().downcallHandle(
                 FunctionDescriptor.of(STRUCT_LL,
-                        Stream.generate(() -> C_DOUBLE).limit(124).toArray(MemoryLayout[]::new)),
+                        Stream.generate(() -> layout).limit(DOWNCALL_AVAILABLE_SLOTS / numSlots).toArray(MemoryLayout[]::new)),
                 Linker.Option.captureCallState("errno"));
     }
 
@@ -62,11 +72,21 @@ public void testDowncallAllowHeap() {
                 Linker.Option.critical(true));
     }
 
-    @Test
-    public void testUpcall() {
+    @ParameterizedTest
+    @MethodSource("layouts")
+    public void testUpcall(ValueLayout layout, int numSlots) {
         // Link a handle with a large number of arguments, to try and overflow the code buffer
         Linker.nativeLinker().downcallHandle(
                 FunctionDescriptor.of(STRUCT_LL,
-                        Stream.generate(() -> C_DOUBLE).limit(125).toArray(MemoryLayout[]::new)));
+                        Stream.generate(() -> layout).limit(UPCALL_AVAILABLE_SLOTS / numSlots).toArray(MemoryLayout[]::new)));
+    }
+
+    private static Stream<Arguments> layouts() {
+        return Stream.of(
+            arguments(C_INT, 1),
+            arguments(C_LONG_LONG, 2),
+            arguments(C_FLOAT, 1),
+            arguments(C_DOUBLE, 2)
+        );
     }
 }