Skip to content

Commit

Permalink
Merge remote-tracking branch 'upstream/master'
Browse files Browse the repository at this point in the history
  • Loading branch information
gluon-bot committed Oct 25, 2023
2 parents 57416ac + 1e9366e commit 27c42f0
Show file tree
Hide file tree
Showing 17 changed files with 533 additions and 47 deletions.
24 changes: 21 additions & 3 deletions sdk/mx.sdk/mx_sdk_vm_impl.py
Original file line number Diff line number Diff line change
Expand Up @@ -1210,7 +1210,10 @@ def is_ee_supported(self):
def is_pgo_supported(self):
return self.is_ee_supported()

def native_image(self, build_args, output_file, allow_server=False, nonZeroIsFatal=True, out=None, err=None):
search_tool = 'strings'
has_search_tool = shutil.which(search_tool) is not None

def native_image(self, build_args, output_file, out=None, err=None, find_bad_strings=False):
assert self._svm_supported
stage1 = get_stage1_graalvm_distribution()
native_image_project_name = GraalVmLauncher.launcher_project_name(mx_sdk.LauncherConfig(mx.exe_suffix('native-image'), [], "", []), stage1=True)
Expand All @@ -1220,7 +1223,22 @@ def native_image(self, build_args, output_file, allow_server=False, nonZeroIsFat
native_image_command += svm_experimental_options([
'-H:Path=' + output_directory or ".",
])
return mx.run(native_image_command, nonZeroIsFatal=nonZeroIsFatal, out=out, err=err)

mx.run(native_image_command, nonZeroIsFatal=True, out=out, err=err)

if find_bad_strings and not mx.is_windows():
if not self.__class__.has_search_tool:
mx.abort(f"Searching for strings requires '{self.__class__.search_tool}' executable.")
try:
strings_in_image = subprocess.check_output([self.__class__.search_tool, output_file], stderr=None).decode().strip().split('\n')
bad_strings = (output_directory, dirname(native_image_bin))
for entry in strings_in_image:
for bad_string in bad_strings:
if bad_string in entry:
mx.abort(f"Found forbidden string '{bad_string}' in native image {output_file}.")

except subprocess.CalledProcessError:
mx.abort(f"Using '{self.__class__.search_tool}' to search for strings in native image {output_file} failed.")

def is_debug_supported(self):
return self._debug_supported
Expand Down Expand Up @@ -2367,7 +2385,7 @@ def build(self):
mx.ensure_dir_exists(dirname(output_file))

# Disable build server (different Java properties on each build prevent server reuse)
self.svm_support.native_image(build_args, output_file)
self.svm_support.native_image(build_args, output_file, find_bad_strings=True)

with open(self._get_command_file(), 'w') as f:
f.writelines((l + os.linesep for l in build_args))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -183,10 +183,21 @@ public void persistOptions(CLongPointer parsedArgs) {

public void verifyOptionValues() {
for (int i = 0; i < OPTION_COUNT; i++) {
validate(OPTIONS[i], getOptionValue(i));
RuntimeOptionKey<?> option = OPTIONS[i];
if (shouldValidate(option)) {
validate(option, getOptionValue(i));
}
}
}

private static boolean shouldValidate(RuntimeOptionKey<?> option) {
if (SubstrateOptions.UseSerialGC.getValue()) {
/* The serial GC supports changing the heap size at run-time to some degree. */
return option != SubstrateGCOptions.MinHeapSize && option != SubstrateGCOptions.MaxHeapSize && option != SubstrateGCOptions.MaxNewSize;
}
return true;
}

@Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
public static boolean getBooleanOptionValue(int index) {
return PARSED_OPTION_VALUES[index] == 1;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
import org.graalvm.nativeimage.VMRuntime;
import org.graalvm.nativeimage.impl.VMRuntimeSupport;

import com.oracle.svm.core.IsolateArgumentParser;
import com.oracle.svm.core.Isolates;
import com.oracle.svm.core.feature.AutomaticallyRegisteredImageSingleton;
import com.oracle.svm.core.heap.HeapSizeVerifier;
Expand Down Expand Up @@ -92,7 +93,7 @@ public boolean isUninitialized() {
public void initialize() {
boolean shouldInitialize = initializationState.compareAndSet(InitializationState.Uninitialized, InitializationState.InProgress);
if (shouldInitialize) {
// GR-35186: we should verify that none of the early parsed isolate arguments changed.
IsolateArgumentParser.singleton().verifyOptionValues();
HeapSizeVerifier.verifyHeapOptions();

executeHooks(startupHooks);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -320,6 +320,11 @@ protected void setOutputKind(List<String> cmd) {
break;
case SHARED_LIBRARY:
cmd.add("-shared");
/*
* Ensure shared library name in image does not use fully qualified build-path
* (GR-46837)
*/
cmd.add("-Wl,-soname=" + outputFile.getFileName());
break;
default:
VMError.shouldNotReachHereUnexpectedInput(imageKind); // ExcludeFromJacocoGeneratedReport
Expand Down
1 change: 1 addition & 0 deletions truffle/CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@ This changelog summarizes major changes between Truffle versions relevant to lan
* Bundle the necessary files into a jar distribution.
* Implement the `InternalResource` interface for handling the resource file unpacking.
* Call the `Env#getInternalResource` when the language or instrument needs the bundled resource files. This method ensures that the requested `InternalResource` is unpacked and provides a directory containing the unpacked files. Since unpacking internal resources can be an expensive operation, the implementation ensures that internal resources are cached.
* GR-44464 Added `TruffleString.ToValidStringNode` for encoding-level string sanitization.

## Version 23.0.0

Expand Down
2 changes: 2 additions & 0 deletions truffle/docs/TruffleStrings.md
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,8 @@ Conversion:
Convert a MutableTruffleString to an immutable TruffleString.
* [AsManaged](https://www.graalvm.org/truffle/javadoc/com/oracle/truffle/api/strings/TruffleString.AsManagedNode.html):
Convert a TruffleString backed by a native pointer to one backed by a java byte array.
* [ToValidString](https://www.graalvm.org/truffle/javadoc/com/oracle/truffle/api/strings/TruffleString.ToValidStringNode.html):
Convert a TruffleString to a version that is encoded correctly.
* [CopyToByteArray](https://www.graalvm.org/truffle/javadoc/com/oracle/truffle/api/strings/TruffleString.CopyToByteArrayNode.html):
Copy a string's content into a byte array.
* [GetInternalByteArray](https://www.graalvm.org/truffle/javadoc/com/oracle/truffle/api/strings/TruffleString.GetInternalByteArrayNode.html):
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,168 @@
/*
* Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* The Universal Permissive License (UPL), Version 1.0
*
* Subject to the condition set forth below, permission is hereby granted to any
* person obtaining a copy of this software, associated documentation and/or
* data (collectively the "Software"), free of charge and under any and all
* copyright rights in the Software, and any and all patent rights owned or
* freely licensable by each licensor hereunder covering either (i) the
* unmodified Software as contributed to or provided by such licensor, or (ii)
* the Larger Works (as defined below), to deal in both
*
* (a) the Software, and
*
* (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if
* one is included with the Software each a "Larger Work" to which the Software
* is contributed by such licensors),
*
* without restriction, including without limitation the rights to copy, create
* derivative works of, display, perform, and distribute the Software and make,
* use, sell, offer for sale, import, export, have made, and have sold the
* Software and the Larger Work(s), and to sublicense the foregoing rights on
* either these or other terms.
*
* This license is subject to the following condition:
*
* The above copyright notice and either this complete permission notice or at a
* minimum a reference to the UPL must be included in all copies or substantial
* portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/

package com.oracle.truffle.api.strings.test.ops;

import static com.oracle.truffle.api.strings.TruffleString.Encoding.BYTES;
import static com.oracle.truffle.api.strings.TruffleString.Encoding.ISO_8859_1;
import static com.oracle.truffle.api.strings.TruffleString.Encoding.US_ASCII;
import static com.oracle.truffle.api.strings.TruffleString.Encoding.UTF_16;
import static com.oracle.truffle.api.strings.TruffleString.Encoding.UTF_32;
import static com.oracle.truffle.api.strings.TruffleString.Encoding.UTF_8;
import static com.oracle.truffle.api.strings.test.TStringTestUtil.byteArray;
import static org.junit.runners.Parameterized.Parameter;

import java.util.Arrays;

import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameters;

import com.oracle.truffle.api.strings.TruffleString;
import com.oracle.truffle.api.strings.test.TStringTestBase;

@RunWith(Parameterized.class)
public class TStringToValidStringTest extends TStringTestBase {

@Parameter public TruffleString.ToValidStringNode node;

@Parameters(name = "{0}")
public static Iterable<TruffleString.ToValidStringNode> data() {
return Arrays.asList(TruffleString.ToValidStringNode.create(), TruffleString.ToValidStringNode.getUncached());
}

@Test
public void testAll() throws Exception {
forAllStrings(new TruffleString.Encoding[]{US_ASCII, ISO_8859_1, BYTES, UTF_8, UTF_16, UTF_32}, true, (a, array, codeRange, isValid, encoding, codepoints, byteIndices) -> {
TruffleString wellFormed = node.execute(a, encoding);
if (isValid && a instanceof TruffleString) {
Assert.assertSame(a, wellFormed);
}
Assert.assertTrue(wellFormed.isValidUncached(encoding));
});
}

@Test
public void testAscii() {
testAscii(byteArray('a', '?'), byteArray('a', 0xff));
testAscii(byteArray('a', '?'), byteArray('a', 0x80));
testAscii(byteArray('a', '?', 'b'), byteArray('a', 0xff, 'b'));
testAscii(byteArray('a', '?', 'b'), byteArray('a', 0x80, 'b'));
testAscii(byteArray('a', 0x7f, 'b'), byteArray('a', 0x7f, 'b'));
}

@Test
public void testUTF8() {
testUTF8(byteArray('a', 0xEF, 0xBF, 0xBD), byteArray('a', 0xff));
testUTF8(byteArray('a', 0xEF, 0xBF, 0xBD), byteArray('a', 0xf0, 0x90));
testUTF8(byteArray('a', 0xEF, 0xBF, 0xBD), byteArray('a', 0xf0, 0x90, 0x80));
testUTF8(byteArray('a', 0xEF, 0xBF, 0xBD, 0xf0, 0x90, 0x80, 0x80), byteArray('a', 0xf0, 0x90, 0x80, 0xf0, 0x90, 0x80, 0x80));
testUTF8(byteArray('a', 0xf0, 0x90, 0x80, 0x80, 0xEF, 0xBF, 0xBD), byteArray('a', 0xf0, 0x90, 0x80, 0x80, 0xf0, 0x90, 0x80));
testUTF8(byteArray('a', 0xf0, 0x90, 0x80, 0x80), byteArray('a', 0xf0, 0x90, 0x80, 0x80));
testUTF8(byteArray('a', 0xEF, 0xBF, 0xBD, 0xEF, 0xBF, 0xBD), byteArray('a', 0xf8, 0x90));
testUTF8(byteArray('a', 0xEF, 0xBF, 0xBD, 'b'), byteArray('a', 0xff, 'b'));
testUTF8(byteArray('a', 0xEF, 0xBF, 0xBD, 'b'), byteArray('a', 0xf0, 0x90, 'b'));
testUTF8(byteArray('a', 0xEF, 0xBF, 0xBD, 'b'), byteArray('a', 0xf0, 0x90, 0x80, 'b'));
testUTF8(byteArray('a', 0xf0, 0x90, 0x80, 0x80, 'b'), byteArray('a', 0xf0, 0x90, 0x80, 0x80, 'b'));
testUTF8(byteArray('a', 0xEF, 0xBF, 0xBD, 0xf0, 0x90, 0x80, 0x80, 'b'), byteArray('a', 0xf0, 0x90, 0x80, 0xf0, 0x90, 0x80, 0x80, 'b'));
testUTF8(byteArray('a', 0xEF, 0xBF, 0xBD, 0xEF, 0xBF, 0xBD, 'b'), byteArray('a', 0xf8, 0x90, 'b'));
}

private void testAscii(byte[] expected, byte[] input) {
testByteArray(expected, input, US_ASCII);
}

private void testUTF8(byte[] expected, byte[] input) {
testByteArray(expected, input, UTF_8);
}

private void testByteArray(byte[] expected, byte[] input, TruffleString.Encoding encoding) {
TruffleString wellFormed = node.execute(TruffleString.fromByteArrayUncached(input, encoding), encoding);
for (int i = 0; i < expected.length; i++) {
Assert.assertEquals(Byte.toUnsignedInt(expected[i]), wellFormed.readByteUncached(i, encoding));
}
Assert.assertTrue(wellFormed.isValidUncached(encoding));
}

@Test
public void testUTF16() {
testUTF16("a\ufffd", "a\udfff");
testUTF16("a\ufffd", "a\udbff");
testUTF16("a\ufffd\ufffd", "a\udfff\udfff");
testUTF16("a\ufffd\ufffd", "a\udbff\udbff");
testUTF16("a\udbff\udfff\ufffd", "a\udbff\udfff\udbff");
testUTF16("a\udbff\udfff\ufffdb", "a\udbff\udfff\udbffb");
}

private void testUTF16(String expected, String input) {
TruffleString wellFormed = node.execute(TruffleString.fromJavaStringUncached(input, UTF_16), UTF_16);
Assert.assertEquals(expected, wellFormed.toJavaStringUncached());
Assert.assertTrue(wellFormed.isValidUncached(UTF_16));
}

@Test
public void testUTF32() {
testUTF32(new int[]{'a', 0xfffd}, new int[]{'a', Character.MIN_SURROGATE});
testUTF32(new int[]{'a', 0xfffd}, new int[]{'a', Character.MAX_SURROGATE});
testUTF32(new int[]{'a', 0xfffd}, new int[]{'a', Integer.MAX_VALUE});
testUTF32(new int[]{'a', 0xfffd}, new int[]{'a', Integer.MIN_VALUE});
testUTF32(new int[]{'a', 0xfffd}, new int[]{'a', 0x110000});
testUTF32(new int[]{'a', 0xfffd}, new int[]{'a', 0xffff_ffff});
testUTF32(new int[]{'a', Character.MAX_CODE_POINT}, new int[]{'a', Character.MAX_CODE_POINT});
testUTF32(new int[]{'a', Character.MAX_CODE_POINT, 0xfffd}, new int[]{'a', Character.MAX_CODE_POINT, Character.MIN_SURROGATE});
}

private void testUTF32(int[] expected, int[] input) {
TruffleString wellFormed = node.execute(TruffleString.fromIntArrayUTF32Uncached(input), UTF_32);
for (int i = 0; i < expected.length; i++) {
Assert.assertEquals(expected[i], wellFormed.codePointAtIndexUncached(i, UTF_32));
}
Assert.assertTrue(wellFormed.isValidUncached(UTF_32));
}

@Test
public void testNull() throws Exception {
expectNullPointerException(() -> node.execute(null, UTF_16));
expectNullPointerException(() -> node.execute(S_UTF16, null));
}
}
Loading

0 comments on commit 27c42f0

Please sign in to comment.