Detect BOM and use respective encoding in org.eclipse.osgi.util.NLS

bundles/org.eclipse.osgi.tests/src/org/eclipse/osgi/tests/util/NLSTestCase.java

@@ -14,6 +14,8 @@
 
 package org.eclipse.osgi.tests.util;
 
+import static org.junit.Assert.assertEquals;
+
 import org.eclipse.osgi.util.NLS;
 import org.junit.Test;
 
@@ -24,4 +26,75 @@ public void testEmptyMessageBug200296() {
 		NLS.bind("", Integer.valueOf(0));
 	}
 
+	@Test
+	public void testPropertiesWithoutBom() {
+		assertEquals("Lorem ipsum dolor sit amet", MessagesNoBom.key_ascii);
+		assertEquals("ÀÅÆÇÈÊËÌÍÏÐÑÒÓÖ×ØÙÚÜÝÞßàâäåæçèéëìíïðñòõö÷øùúüýþ", MessagesNoBom.key_latin1);
+	}
+
+	@Test
+	public void testPropertiesBomUtf8() {
+		assertEquals("Ḽơᶉëᶆ ȋṕšᶙṁ", MessagesBomUtf8.key_en);
+		assertEquals("顾客很高兴", MessagesBomUtf8.key_ch);
+		assertEquals("고객은 매우 행복합니다", MessagesBomUtf8.key_ko);
+		assertEquals("Клиент очень доволен", MessagesBomUtf8.key_ru);
+	}
+
+	@Test
+	public void testPropertiesBomUtf16Be() {
+		assertEquals("Ḽơᶉëᶆ ȋṕšᶙṁ", MessagesBomUtf16Be.key_en);
+		assertEquals("顾客很高兴", MessagesBomUtf16Be.key_ch);
+		assertEquals("고객은 매우 행복합니다", MessagesBomUtf16Be.key_ko);
+		assertEquals("Клиент очень доволен", MessagesBomUtf16Be.key_ru);
+	}
+
+	@Test
+	public void testPropertiesBomUtf16Le() {
+		assertEquals("Ḽơᶉëᶆ ȋṕšᶙṁ", MessagesBomUtf16Le.key_en);
+		assertEquals("顾客很高兴", MessagesBomUtf16Le.key_ch);
+		assertEquals("고객은 매우 행복합니다", MessagesBomUtf16Le.key_ko);
+		assertEquals("Клиент очень доволен", MessagesBomUtf16Le.key_ru);
+	}
+
+	public static class MessagesNoBom extends NLS {
+		public static String key_ascii;
+		public static String key_latin1;
+
+		static {
+			NLS.initializeMessages("org.eclipse.osgi.tests.util.nls.messages_no_bom", MessagesNoBom.class);
+		}
+	}
+
+	public static class MessagesBomUtf8 extends NLS {
+		public static String key_en;
+		public static String key_ch;
+		public static String key_ko;
+		public static String key_ru;
+
+		static {
+			NLS.initializeMessages("org.eclipse.osgi.tests.util.nls.messages_bom_utf8", MessagesBomUtf8.class);
+		}
+	}
+
+	public static class MessagesBomUtf16Be extends NLS {
+		public static String key_en;
+		public static String key_ch;
+		public static String key_ko;
+		public static String key_ru;
+
+		static {
+			NLS.initializeMessages("org.eclipse.osgi.tests.util.nls.messages_bom_utf16be", MessagesBomUtf16Be.class);
+		}
+	}
+
+	public static class MessagesBomUtf16Le extends NLS {
+		public static String key_en;
+		public static String key_ch;
+		public static String key_ko;
+		public static String key_ru;
+
+		static {
+			NLS.initializeMessages("org.eclipse.osgi.tests.util.nls.messages_bom_utf16le", MessagesBomUtf16Le.class);
+		}
+	}
 }

bundles/org.eclipse.osgi.tests/src/org/eclipse/osgi/tests/util/nls/messages_bom_utf8.properties

@@ -0,0 +1,5 @@
+key_en=Ḽơᶉëᶆ ȋṕšᶙṁ
+# This is a comment これはコメントです
+key_ch=顾客很高兴
+key_ko=고객은 매우 행복합니다
+key_ru=Клиент очень доволен

bundles/org.eclipse.osgi.tests/src/org/eclipse/osgi/tests/util/nls/messages_no_bom.properties

@@ -0,0 +1,3 @@
+key_ascii=Lorem ipsum dolor sit amet
+# This is a comment
+key_latin1=ÀÅÆÇÈÊËÌÍÏÐÑÒÓÖ×ØÙÚÜÝÞßàâäåæçèéëìíïðñòõö÷øùúüýþ

bundles/org.eclipse.osgi/META-INF/MANIFEST.MF

@@ -51,7 +51,7 @@ Export-Package: org.eclipse.core.runtime.adaptor;x-friends:="org.eclipse.core.ru
  org.eclipse.osgi.storage.bundlefile;x-internal:=true,
  org.eclipse.osgi.storage.url.reference;x-internal:=true,
  org.eclipse.osgi.storagemanager;version="1.0",
- org.eclipse.osgi.util;version="1.2.0",
+ org.eclipse.osgi.util;version="1.3",
  org.osgi.dto;version="1.1.1",
  org.osgi.framework;version="1.10",
  org.osgi.framework.connect;version="1.0";uses:="org.osgi.framework.launch",
@@ -107,7 +107,7 @@ Bundle-Activator: org.eclipse.osgi.internal.framework.SystemBundleActivator
 Bundle-Description: %systemBundle
 Bundle-Copyright: %copyright
 Bundle-Vendor: %eclipse.org
-Bundle-Version: 3.22.100.qualifier
+Bundle-Version: 3.23.0.qualifier
 Bundle-Localization: systembundle
 Bundle-DocUrl: http://www.eclipse.org
 Eclipse-ExtensibleAPI: true

bundles/org.eclipse.osgi/supplement/META-INF/MANIFEST.MF

@@ -2,7 +2,7 @@ Manifest-Version: 1.0
 Bundle-ManifestVersion: 2
 Bundle-Name: %pluginName
 Bundle-SymbolicName: org.eclipse.equinox.supplement
-Bundle-Version: 1.11.100.qualifier
+Bundle-Version: 1.12.0.qualifier
 Bundle-Vendor: %providerName
 Bundle-Localization: plugin
 Export-Package: org.eclipse.equinox.log;version="1.1",
@@ -17,7 +17,7 @@ Export-Package: org.eclipse.equinox.log;version="1.1",
  org.eclipse.osgi.service.runnable;version="1.1",
  org.eclipse.osgi.service.urlconversion;version="1.0",
  org.eclipse.osgi.storagemanager;version="1.0",
- org.eclipse.osgi.util;version="1.1"
+ org.eclipse.osgi.util;version="1.2"
 Bundle-RequiredExecutionEnvironment: JavaSE-1.8
 Import-Package: org.osgi.framework,
  org.osgi.framework.hooks.resolver,

bundles/org.eclipse.osgi/supplement/src/org/eclipse/osgi/util/NLS.java

@@ -13,10 +13,15 @@
  *******************************************************************************/
 package org.eclipse.osgi.util;
 
+import java.io.BufferedInputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.Reader;
 import java.lang.reflect.Field;
 import java.lang.reflect.Modifier;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
 import java.security.AccessController;
 import java.security.PrivilegedAction;
 import java.util.ArrayList;
@@ -347,7 +352,7 @@ static void load(final String bundleName, Class<?> clazz) {
 				continue;
 			try {
 				final MessagesProperties properties = new MessagesProperties(fields, bundleName, isAccessible);
-				properties.load(input);
+				properties.load(newBomReader(input));
 			} catch (IOException e) {
 				log(SEVERITY_ERROR, "Error loading " + variant, e); //$NON-NLS-1$
 			} finally {
@@ -402,6 +407,35 @@ static void log(int severity, String message, Exception e) {
 			e.printStackTrace();
 	}
 
+	private static Reader newBomReader(InputStream input) throws IOException {
+		BufferedInputStream is = new BufferedInputStream(input);
+		Charset cs = consumeBom(is);
+		if (cs == null) {
+			cs = StandardCharsets.ISO_8859_1;
+		}
+		return new InputStreamReader(is, cs);
+	}
+
+	private static Charset consumeBom(BufferedInputStream is) throws IOException {
+		is.mark(3);
+		switch (is.read()) {
+		case 0xEF: // EF BB BF
+				if (is.read() == 0xBB && is.read() == 0xBF)
+					return StandardCharsets.UTF_8;
+				break;
+			case 0xFE: // FE FF
+				if (is.read() == 0xFF)
+					return StandardCharsets.UTF_16BE;
+				break;
+			case 0xFF: // FF FE
+				if (is.read() == 0xFE)
+					return StandardCharsets.UTF_16LE;
+				break;
+			}
+			is.reset();
+			return null;
+		}
+
 	/*
 	 * Class which sub-classes java.util.Properties and uses the #put method
 	 * to set field values rather than storing the values in the table.