From 3db203c37b6efb5ad3bcf14910d58a534adb8b65 Mon Sep 17 00:00:00 2001
From: Peter Shipton <Peter_Shipton@ca.ibm.com>
Date: Thu, 21 Sep 2023 09:02:45 -0400
Subject: [PATCH] Look for -XX:[+/-]UseZlibNX on AIX

AIX puts zlibNX on the LIBPATH when running on P9 or later, but having
this in the environment may cause some things, such as git clone, to
fail when exec'ed from Java. For example,
`fatal: pack has bad object at offset 3872100: inflate returned -5`

Backport of https://github.com/ibmruntimes/openj9-openjdk-jdk/pull/664

Signed-off-by: Peter Shipton <Peter_Shipton@ca.ibm.com>
---
 src/java.base/unix/native/libjli/java_md.c | 33 ++++++++++++++++++----
 1 file changed, 28 insertions(+), 5 deletions(-)

diff --git a/src/java.base/unix/native/libjli/java_md.c b/src/java.base/unix/native/libjli/java_md.c
index e4f63284fd4..9c30d4f2c3c 100644
--- a/src/java.base/unix/native/libjli/java_md.c
+++ b/src/java.base/unix/native/libjli/java_md.c
@@ -25,7 +25,7 @@
 
 /*
  * ===========================================================================
- * (c) Copyright IBM Corp. 2020, 2021 All Rights Reserved
+ * (c) Copyright IBM Corp. 2020, 2023 All Rights Reserved
  * ===========================================================================
  */
 
@@ -376,12 +376,35 @@ CreateExecutionEnvironment(int *pargc, char ***pargv,
          *     o          $JVMPATH (directory portion only)
          *     o          $JRE/lib
          *     o          $JRE/../lib
-         *     o          ZLIBNX_PATH (for AIX P9 or newer systems with NX)
+         *     o          ZLIBNX_PATH (for AIX P9 or newer systems with NX, unless -XX:-UseZlibNX is set)
          *
          * followed by the user's previous effective LD_LIBRARY_PATH, if
          * any.
          */
 
+#ifdef AIX
+        int aixargc = *pargc - 1; // skip the launcher name
+        char **aixargv = *pargv + 1;
+        const char *aixarg = NULL;
+        jboolean useZlibNX = JNI_TRUE;
+        while (aixargc > 0 && *(aixarg = *aixargv) == '-') {
+            if (JLI_StrCmp(aixarg, "-XX:+UseZlibNX") == 0) {
+                useZlibNX = JNI_TRUE;
+            } else if (JLI_StrCmp(aixarg, "-XX:-UseZlibNX") == 0) {
+                useZlibNX = JNI_FALSE;
+            }
+            aixargc--;
+            aixargv++;
+        }
+        useZlibNX = useZlibNX && power_9_andup() && power_nx_gzip();
+        if (JLI_IsTraceLauncher()) {
+            printf("Add " ZLIBNX_PATH " to the LIBPATH: %s P9+ %s NX %s\n",
+                useZlibNX ? "TRUE" : "FALSE",
+                power_9_andup() ? "TRUE" : "FALSE",
+                power_nx_gzip() ? "TRUE" : "FALSE");
+        }
+#endif
+
         runpath = getenv(LD_LIBRARY_PATH);
 
         /* runpath contains current effective LD_LIBRARY_PATH setting */
@@ -390,8 +413,8 @@ CreateExecutionEnvironment(int *pargc, char ***pargv,
             new_runpath_size = ((runpath != NULL) ? JLI_StrLen(runpath) : 0) +
                     2 * JLI_StrLen(jrepath) +
 #ifdef AIX
-                    /* On AIX P9 or newer with NX accelerator enabled, add the accelerated zlibNX to LIBPATH */
-                    ((power_9_andup() && power_nx_gzip()) ? JLI_StrLen(":" ZLIBNX_PATH) : 0) +
+                    /* On AIX P9 or newer with NX accelerator enabled, add the accelerated zlibNX to LIBPATH. */
+                    (useZlibNX ? JLI_StrLen(":" ZLIBNX_PATH) : 0) +
 #endif
                     JLI_StrLen(new_jvmpath) + 52;
             new_runpath = JLI_MemAlloc(new_runpath_size);
@@ -419,7 +442,7 @@ CreateExecutionEnvironment(int *pargc, char ***pargv,
                         jrepath,
                         jrepath
 #ifdef AIX
-                        , ((power_9_andup() && power_nx_gzip()) ? (":" ZLIBNX_PATH) : "")
+                        , (useZlibNX ? (":" ZLIBNX_PATH) : "")
 #endif
                         );